public async Task ThenIfFileHasMissingFieldsReturnError(string header)
{
var inputData = $"{header}" +
@"
Abba123,1113335559,Froberg,Chris,1998-12-08,SE123321C,25,,,2,2120-08,2125-08,1500,,Employer ref,Provider ref";
var textStream = new MemoryStream(UTF8.GetBytes(inputData));
_file.Setup(m => m.InputStream).Returns(textStream);
BulkUploadApprenticeshipsCommand commandArgument = null;
_mockMediator.Setup(x => x.Send(It.IsAny <BulkUploadApprenticeshipsCommand>(), new CancellationToken()))
.ReturnsAsync(new Unit())
.Callback((object x) => commandArgument = x as BulkUploadApprenticeshipsCommand);
_mockMediator.Setup(m => m.Send(It.IsAny <GetCommitmentQueryRequest>(), new CancellationToken()))
.Returns(Task.FromResult(new GetCommitmentQueryResponse
{
Commitment = new CommitmentView
{
AgreementStatus = AgreementStatus.NotAgreed,
EditStatus = EditStatus.ProviderOnly
}
}));
_mockMediator.Setup(m => m.Send(It.IsAny <GetOverlappingApprenticeshipsQueryRequest>(), It.IsAny <CancellationToken>()))
.Returns(
Task.Run(() => new GetOverlappingApprenticeshipsQueryResponse
{
Overlaps = new List <ApprenticeshipOverlapValidationResult>
{
new ApprenticeshipOverlapValidationResult
{
OverlappingApprenticeships = new List <OverlappingApprenticeship>
{
new OverlappingApprenticeship
{
Apprenticeship = new Apprenticeship {
ULN = "1113335559"
},
ValidationFailReason = ValidationFailReason.DateEmbrace
}
}
}
}
}));
var model = new UploadApprenticeshipsViewModel {
Attachment = _file.Object, HashedCommitmentId = "ABBA123", ProviderId = 111
};
var file = await _sut.UploadFile("user123", model, new SignInUserModel());
//Assert
Assert.IsTrue(file.HasFileLevelErrors);
_logger.Verify(x => x.Info(It.IsAny <string>(), It.IsAny <long?>(), It.IsAny <long?>(), It.IsAny <long?>()), Times.Once);
_logger.Verify(x => x.Error(It.IsAny <Exception>(), It.IsAny <string>(), It.IsAny <long?>(), It.IsAny <long?>(), It.IsAny <long?>()), Times.Never);
}
public async Task ThenIfAnyRecordsOverlapWithActiveApprenticeshipsThenReturnError()
{
const string dataLine = "\n\rABBA123,Chris,Froberg,1998-12-08,,,25,2,2020-08,2025-08,1500,,Employer ref,Provider ref,1113335559";
const string fileContents = HeaderLine + dataLine;
var textStream = new MemoryStream(UTF8.GetBytes(fileContents));
_file.Setup(m => m.InputStream).Returns(textStream);
BulkUploadApprenticeshipsCommand commandArgument = null;
_mockMediator.Setup(x => x.Send(It.IsAny <BulkUploadApprenticeshipsCommand>(), new CancellationToken()))
.ReturnsAsync(new Unit())
.Callback((object x) => commandArgument = x as BulkUploadApprenticeshipsCommand);
_mockMediator.Setup(m => m.Send(It.IsAny <GetCommitmentQueryRequest>(), new CancellationToken()))
.Returns(Task.FromResult(new GetCommitmentQueryResponse
{
Commitment = new CommitmentView
{
AgreementStatus = AgreementStatus.NotAgreed,
EditStatus = EditStatus.ProviderOnly
}
}));
_mockMediator.Setup(m => m.Send(It.IsAny <GetOverlappingApprenticeshipsQueryRequest>(), It.IsAny <CancellationToken>()))
.Returns(
Task.Run(() => new GetOverlappingApprenticeshipsQueryResponse
{
Overlaps = new List <ApprenticeshipOverlapValidationResult>
{
new ApprenticeshipOverlapValidationResult
{
OverlappingApprenticeships = new List <OverlappingApprenticeship>
{
new OverlappingApprenticeship
{
Apprenticeship = new Apprenticeship {
ULN = "1113335559"
},
ValidationFailReason = ValidationFailReason.DateEmbrace
}
}
}
}
}));
var model = new UploadApprenticeshipsViewModel {
Attachment = _file.Object, HashedCommitmentId = "ABBA123", ProviderId = 111
};
var file = await _sut.UploadFile("user123", model, new SignInUserModel());
//Assert
Assert.IsTrue(file.HasRowLevelErrors);
}
public async Task ShouldCallMediatorPassingInMappedApprenticeships()
{
const string dataLine = "\n\rABBA123,Chris,Froberg,1998-12-08,,,25,2,2020-08,2025-08,1500,,Employer ref,Provider ref,1113335559";
const string fileContents = HeaderLine + dataLine;
var textStream = new MemoryStream(UTF8.GetBytes(fileContents));
_file.Setup(m => m.InputStream).Returns(textStream);
BulkUploadApprenticeshipsCommand commandArgument = null;
_mockMediator.Setup(x => x.Send(It.IsAny <BulkUploadApprenticeshipsCommand>(), new CancellationToken()))
.ReturnsAsync(new Unit())
.Callback <BulkUploadApprenticeshipsCommand, CancellationToken>((command, token) => commandArgument = command);
_mockMediator.Setup(m => m.Send(It.IsAny <GetCommitmentQueryRequest>(), new CancellationToken()))
.Returns(Task.FromResult(new GetCommitmentQueryResponse
{
Commitment = new CommitmentView
{
AgreementStatus = AgreementStatus.NotAgreed,
EditStatus = EditStatus.ProviderOnly
}
}));
var model = new UploadApprenticeshipsViewModel {
Attachment = _file.Object, HashedCommitmentId = "ABBA123", ProviderId = 111
};
var signinUser = new SignInUserModel {
DisplayName = "Bob", Email = "*****@*****.**"
};
await _sut.UploadFile("user123", model, signinUser);
_mockMediator.Verify(x => x.Send(It.IsAny <BulkUploadApprenticeshipsCommand>(), new CancellationToken()), Times.Once);
commandArgument.ProviderId.Should().Be(111);
commandArgument.CommitmentId.Should().Be(123);
commandArgument.Apprenticeships.Should().NotBeEmpty();
commandArgument.Apprenticeships.ToList()[0].FirstName.Should().Be("Chris");
commandArgument.Apprenticeships.ToList()[0].LastName.Should().Be("Froberg");
commandArgument.Apprenticeships.ToList()[0].DateOfBirth.Should().Be(new DateTime(1998, 12, 8));
commandArgument.Apprenticeships.ToList()[0].TrainingType.Should().Be(DAS.Commitments.Api.Types.Apprenticeship.Types.TrainingType.Standard);
commandArgument.Apprenticeships.ToList()[0].TrainingCode.Should().Be("2");
commandArgument.Apprenticeships.ToList()[0].StartDate.Should().Be(new DateTime(2020, 8, 1));
commandArgument.Apprenticeships.ToList()[0].EndDate.Should().Be(new DateTime(2025, 8, 1));
commandArgument.Apprenticeships.ToList()[0].Cost.Should().Be(1500);
commandArgument.Apprenticeships.ToList()[0].ProviderRef.Should().Be("Provider ref");
commandArgument.Apprenticeships.ToList()[0].ULN.Should().Be("1113335559");
commandArgument.UserEmailAddress.Should().Be(signinUser.Email);
commandArgument.UserDisplayName.Should().Be(signinUser.DisplayName);
commandArgument.UserId.Should().Be("user123");
}
public async Task TestPerformance()
{
_mockMediator.Setup(m => m.Send(It.IsAny <GetCommitmentQueryRequest>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(new GetCommitmentQueryResponse
{
Commitment = new CommitmentView
{
AgreementStatus = AgreementStatus.NotAgreed,
EditStatus = EditStatus.ProviderOnly
}
}));
const int upper = 40 * 1000;
var testData = new List <string>();
for (int i = 0; i < upper; i++)
{
var uln = (1000000000 + i).ToString();
testData.Add("\n\rABBA123,Chris,Froberg,1998-12-08,,,,2,2020-08,2025-08,1500,,Employer ref,Provider ref," + uln);
}
var str = HeaderLine + string.Join("", testData);
var textStream = new MemoryStream(UTF8.GetBytes(str));
_file.Setup(m => m.InputStream).Returns(textStream);
var model = new UploadApprenticeshipsViewModel {
Attachment = _file.Object, HashedCommitmentId = "ABBA123", ProviderId = 1234L
};
var stopwatch = Stopwatch.StartNew();
var r1 = await _sut.UploadFile("user123", model, new SignInUserModel());
stopwatch.Stop(); Console.WriteLine($"Time TOTAL: {stopwatch.Elapsed.Seconds}");
r1.RowLevelErrors.Count().Should().Be(80 * 1000);
stopwatch.Elapsed.Seconds.Should().BeLessThan(7);
}
public async Task <ActionResult> UploadApprenticeships(UploadApprenticeshipsViewModel model)
{
if (!ModelState.IsValid)
{
return(View(model));
}
var result = await _bulkUploadOrchestrator.UploadFile(CurrentUserId, model, GetSignedInUser());
if (result.HasFileLevelErrors)
{
var error = result.FileLevelErrors.FirstOrDefault();
ModelState.AddModelError("Attachment", error?.Message);
return(View(model));
}
if (result.HasRowLevelErrors)
{
return(RedirectToAction("UploadApprenticeshipsUnsuccessful", new { model.ProviderId, model.HashedCommitmentId, result.BulkUploadReference }));
}
// ToDo: Flash message, or other feedback to customer
return(RedirectToAction("Details", "Commitment", new { model.ProviderId, model.HashedCommitmentId }));
}
public async Task <BulkUploadResultViewModel> UploadFile(string userId, UploadApprenticeshipsViewModel uploadApprenticeshipsViewModel, SignInUserModel signInUser)
{
var commitmentId = HashingService.DecodeValue(uploadApprenticeshipsViewModel.HashedCommitmentId);
var providerId = uploadApprenticeshipsViewModel.ProviderId;
var fileName = uploadApprenticeshipsViewModel.Attachment?.FileName ?? "<unknown>";
var commitment = await GetCommitment(providerId, commitmentId);
AssertCommitmentStatus(commitment);
await AssertAutoReservationEnabled(commitment);
Logger.Info($"Uploading File - Filename:{fileName}", uploadApprenticeshipsViewModel.ProviderId, commitmentId);
var fileValidationResult = await _bulkUploader.ValidateFileStructure(uploadApprenticeshipsViewModel, providerId, commitment);
if (fileValidationResult.Errors.Any())
{
return(new BulkUploadResultViewModel
{
BulkUploadId = fileValidationResult.BulkUploadId,
HasFileLevelErrors = true,
FileLevelErrors = fileValidationResult.Errors
});
}
Logger.Info("Uploading file of apprentices.", providerId, commitmentId);
var rowValidationResult = await _bulkUploader.ValidateFileRows(fileValidationResult.Data, providerId, fileValidationResult.BulkUploadId);
var sw = Stopwatch.StartNew();
var overlapErrors = await GetOverlapErrors(fileValidationResult.Data.ToList());
Logger.Trace($"Validating overlaps took {sw.ElapsedMilliseconds}");
var rowErrors = rowValidationResult.Errors.ToList();
rowErrors.AddRange(overlapErrors);
var hashedBulkUploadId = HashingService.HashValue(fileValidationResult.BulkUploadId);
if (rowErrors.Any())
{
Logger.Info($"{rowErrors.Count} Upload errors", providerId, commitmentId);
return(new BulkUploadResultViewModel
{
BulkUploadId = fileValidationResult.BulkUploadId,
BulkUploadReference = hashedBulkUploadId,
HasRowLevelErrors = true,
RowLevelErrors = rowErrors
});
}
try
{
await Mediator.Send(new BulkUploadApprenticeshipsCommand
{
UserId = userId,
ProviderId = providerId,
CommitmentId = commitmentId,
Apprenticeships = await _mapper.MapFrom(commitmentId, rowValidationResult.Data),
UserEmailAddress = signInUser.Email,
UserDisplayName = signInUser.DisplayName
});
}
catch (Exception)
{
var overlaps = (await GetOverlapErrors(fileValidationResult.Data.ToList())).ToList();
if (overlaps.Any())
{
return(new BulkUploadResultViewModel
{
BulkUploadId = fileValidationResult.BulkUploadId,
HasRowLevelErrors = true,
RowLevelErrors = overlaps
});
}
throw;
}
return(new BulkUploadResultViewModel {
BulkUploadId = fileValidationResult.BulkUploadId
});
}
public async Task <BulkUploadResult> ValidateFileStructure(UploadApprenticeshipsViewModel uploadApprenticeshipsViewModel, long providerId, CommitmentView commitment)
{
if (uploadApprenticeshipsViewModel.Attachment == null)
{
return new BulkUploadResult {
Errors = new List <UploadError> {
new UploadError("No file chosen")
}
}
}
;
var fileContent = new StreamReader(uploadApprenticeshipsViewModel.Attachment.InputStream).ReadToEnd();
var fileName = uploadApprenticeshipsViewModel?.Attachment?.FileName ?? "<- NO NAME ->";
_logger.Trace($"Saving bulk upload file. {fileName}");
var bulkUploadId = await _mediator.Send(
new SaveBulkUploadFileCommand
{
ProviderId = uploadApprenticeshipsViewModel.ProviderId,
CommitmentId = commitment.Id,
FileContent = fileContent,
FileName = fileName
});
_logger.Info($"Saved bulk upload with Id: {bulkUploadId}");
var fileAttributeErrors = _bulkUploadValidator.ValidateFileSize(uploadApprenticeshipsViewModel.Attachment).ToList();
if (fileAttributeErrors.Any())
{
foreach (var error in fileAttributeErrors)
{
_logger.Warn($"File Structure Error --> {error.Message}", uploadApprenticeshipsViewModel.ProviderId, commitment.Id);
}
_logger.Info($"Failed validation bulk upload file with {fileAttributeErrors.Count} errors", uploadApprenticeshipsViewModel.ProviderId, commitment.Id);
return(new BulkUploadResult {
Errors = fileAttributeErrors
});
}
var uploadResult = _fileParser.CreateViewModels(providerId, commitment, fileContent);
if (uploadResult.HasErrors)
{
return(uploadResult);
}
var errors = _bulkUploadValidator.ValidateCohortReference(uploadResult.Data, uploadApprenticeshipsViewModel.HashedCommitmentId).ToList();
errors.AddRange(_bulkUploadValidator.ValidateUlnUniqueness(uploadResult.Data).ToList());
return(new BulkUploadResult
{
Errors = errors,
Data = uploadResult.Data,
BulkUploadId = bulkUploadId
});
}
