public void ShouldAllowBulkLoadApiKeyAssociatedWithValidApplication()
{
var odsInstance1 = new OdsInstance
{
Name = "Test Instance 1",
InstanceType = "Ods",
IsExtended = false,
Status = "OK",
Version = "1.0.0"
};
var apiClientForInstance1 = SetupApplicationForInstance(odsInstance1);
var bulkFileUpLoadModel = new BulkFileUploadModel
{
ApiKey = apiClientForInstance1.Key
};
Scoped <IUsersContext>(usersContext =>
{
var validator = new BulkFileUploadModelValidator(usersContext);
var validationResults = validator.Validate(bulkFileUpLoadModel);
validationResults.IsValid.ShouldBe(true);
});
}
public ActionResult BulkFileUpload(BulkFileUploadModel model)
{
var bulkFiles = model.BulkFiles.Where(file => file != null && file.ContentLength > 0).ToArray();
if (!bulkFiles.Any())
{
return(new HttpStatusCodeResult(HttpStatusCode.NoContent));
}
if (bulkFiles.Sum(f => f.ContentLength) > BulkFileUploadModel.MaxFileSize)
{
throw new Exception($"Upload exceeds maximum limit of {BulkFileUploadModel.MaxFileSize} bytes");
}
if (bulkFiles.Length > 1)
{
throw new Exception("Currently, the bulk import process only supports a single file at a time");
}
var uploadedFiles = _fileUploadHandler.SaveFilesToUploadDirectory(bulkFiles, fileName => InterchangeFileHelpers.BuildFileNameForImport(model.BulkFileType, fileName));
var jobContext = new BulkUploadJobContext
{
DataDirectoryFullPath = uploadedFiles.Directory
};
_bulkUploadJob.EnqueueJob(jobContext);
return(new HttpStatusCodeResult(HttpStatusCode.Accepted));
}
public void When_Perform_Post_Request_To_BulkFileUpload_With_No_File_ThrowsException()
{
// Arrange
var model = new BulkFileUploadModel
{
BulkFiles = new List <IFormFile>()
};
// Assert
Assert.ThrowsAsync <Exception>(() => SystemUnderTest.BulkFileUpload(model)).Message
.Contains("The given file is empty. Please upload a file compatible with the Ed-Fi Data Standard.").ShouldBeTrue();
}
public void When_Perform_Post_Request_To_BulkFileUpload_With_Greater_File_ContentLength_ThrowsException()
{
// Arrange
var file = new Mock <IFormFile>();
file.Setup(x => x.Length).Returns(20000002);
var model = new BulkFileUploadModel
{
BulkFiles = new List <IFormFile>
{
file.Object
}
};
// Assert
Assert.ThrowsAsync <Exception>(() => SystemUnderTest.BulkFileUpload(model)).Message
.Contains("Upload exceeds maximum limit").ShouldBeTrue();
}
public void ShouldNotAllowBulkLoadApiKeyWhichNotAssociatedWithValidApplication()
{
var bulkFileUpLoadModel = new BulkFileUploadModel
{
ApiKey = $"RandomKey-{Guid.NewGuid()}"
};
Scoped <IUsersContext>(usersContext =>
{
var validator = new BulkFileUploadModelValidator(usersContext);
var validationResults = validator.Validate(bulkFileUpLoadModel);
validationResults.IsValid.ShouldBe(false);
validationResults.Errors.Select(x => x.ErrorMessage).ShouldBe(new List <string>
{
"Provided Api Key is not associated with any application. Please reset the credentials."
});
});
}
private BulkFileUploadModel SetupBulkUpload(out FileUploadResult fileUploadResult)
{
const string filename = "test.xml";
var file = new Mock <IFormFile>();
file.Setup(x => x.Length).Returns(200);
file.Setup(x => x.FileName).Returns("test.xml");
var model = new BulkFileUploadModel
{
BulkFileType = InterchangeFileType.AssessmentMetadata.Value,
BulkFiles = new List <IFormFile>
{
file.Object
}
};
fileUploadResult = new FileUploadResult
{
Directory = "directoryPath",
FileNames = new[] { filename }
};
InstanceContext.Id = OdsInstanceContext.Id;
InstanceContext.Name = OdsInstanceContext.Name;
FileUploadHandler.Setup(x =>
x.SaveFilesToUploadDirectory(It.IsAny <IFormFile[]>(), It.IsAny <Func <string, string> >(), WebHostingEnvironment.Object))
.Returns(fileUploadResult);
ApiConnectionInformationProvider
.Setup(x => x.GetConnectionInformationForEnvironment())
.ReturnsAsync(_connectionInformation);
OdsSecretConfigurationProvider.Setup(x => x.GetSecretConfiguration(It.IsAny <int>()))
.Returns(Task.FromResult(OdsSecretConfig));
return(model);
}
public void When_Perform_Post_Request_To_BulkFileUpload_With_Multiple_Files_ThrowsException()
{
// Arrange
var file1 = new Mock <IFormFile>();
file1.Setup(x => x.Length).Returns(200);
var file2 = new Mock <IFormFile>();
file2.Setup(x => x.Length).Returns(200);
var model = new BulkFileUploadModel
{
BulkFiles = new List <IFormFile>
{
file1.Object, file2.Object
}
};
// Assert
Assert.ThrowsAsync <Exception>(() => SystemUnderTest.BulkFileUpload(model)).Message
.Contains("Currently, the bulk import process only supports a single file at a time").ShouldBeTrue();
}
public async Task <ActionResult> BulkFileUpload(BulkFileUploadModel model)
{
var bulkFiles = model.BulkFiles.Where(file => file != null && file.Length > 0).ToArray();
if (!bulkFiles.Any())
{
throw new Exception("The given file is empty. Please upload a file compatible with the Ed-Fi Data Standard.");
}
if (bulkFiles.Sum(f => f.Length) > BulkFileUploadModel.MaxFileSize)
{
throw new Exception($"Upload exceeds maximum limit of {BulkFileUploadModel.MaxFileSize} bytes");
}
if (bulkFiles.Length > 1)
{
throw new Exception("Currently, the bulk import process only supports a single file at a time");
}
InterchangeFileType bulkFileUploadType = null;
if (model.BulkFileType != null)
{
bulkFileUploadType = InterchangeFileType.FromInt32(model.BulkFileType.Value);
}
var uploadedFiles = _fileUploadHandler.SaveFilesToUploadDirectory(bulkFiles,
fileName => InterchangeFileHelpers.BuildFileNameForImport(bulkFileUploadType, fileName), _webHostEnvironment);
var connectionInformation = await GetConnectionInformationProvider();
var config = await _odsSecretConfigurationProvider.GetSecretConfiguration(_instanceContext.Id);
if (config == null)
{
throw new InvalidOperationException("ODS secret configuration can not be null.");
}
if (model != null)
{
model.ApiKey = config.BulkUploadCredential?.ApiKey;
var validationResult = await _bulkLoadValidator.ValidateAsync(model);
if (!validationResult.IsValid)
{
var errorMessage = string.Join(",", validationResult.Errors.Select(x => x.ErrorMessage));
Response.StatusCode = (int)HttpStatusCode.BadRequest;
return(Json(new { Result = new { Errors = new[] { new { ErrorMessage = errorMessage } } } }));
}
}
var schemaBasePath = Path.Combine(_webHostEnvironment.ContentRootPath, _appSettings.XsdFolder);
var standardVersion = await _inferOdsApiVersion.EdFiStandardVersion(connectionInformation.ApiServerUrl);
var odsApiVersion = await _inferOdsApiVersion.Version(connectionInformation.ApiServerUrl);
const int IdealSimultaneousRequests = 20;
const int PessimisticSimultaneousRequests = 1;
var maxSimultaneousRequests = odsApiVersion.StartsWith("3.")
? PessimisticSimultaneousRequests
: IdealSimultaneousRequests;
var jobContext = new BulkUploadJobContext
{
ApiServerUrl = connectionInformation.ApiServerUrl,
DataDirectoryFullPath = uploadedFiles.Directory,
OdsInstanceId = _instanceContext.Id,
OdsInstanceName = _instanceContext.Name,
ApiBaseUrl = connectionInformation.ApiBaseUrl,
OauthUrl = connectionInformation.OAuthUrl,
MetadataUrl = connectionInformation.MetadataUrl,
DependenciesUrl = connectionInformation.DependenciesUrl,
ClientKey = config.BulkUploadCredential?.ApiKey ?? string.Empty,
ClientSecret = config.BulkUploadCredential?.ApiSecret ?? string.Empty,
SchemaPath = Path.Combine(schemaBasePath, standardVersion),
MaxSimultaneousRequests = maxSimultaneousRequests
};
if (!_bulkUploadJob.IsJobRunning())
{
_bulkUploadJob.EnqueueJob(jobContext);
}
var updatedValue = model;
updatedValue.IsJobRunning = _bulkUploadJob.IsJobRunning();
updatedValue.IsSameOdsInstance = _bulkUploadJob.IsSameOdsInstance(_instanceContext.Id, typeof(BulkUploadJobContext));
return(PartialView("_SignalRStatus_BulkLoad", updatedValue));
}
