public async Task <ActionResult> BulkFileUpload(OdsInstanceSettingsModel model)
{
var bulkFiles = model.BulkFileUploadModel.BulkFiles.Where(file => file != null && file.ContentLength > 0).ToArray();
if (!bulkFiles.Any())
{
return(new HttpStatusCodeResult(HttpStatusCode.NoContent));
}
if (bulkFiles.Sum(f => f.ContentLength) > BulkFileUploadModel.MaxFileSize)
{
throw new Exception($"Upload exceeds maximum limit of {BulkFileUploadModel.MaxFileSize} bytes");
}
if (bulkFiles.Length > 1)
{
throw new Exception("Currently, the bulk import process only supports a single file at a time");
}
var uploadedFiles = _fileUploadHandler.SaveFilesToUploadDirectory(bulkFiles,
fileName => InterchangeFileHelpers.BuildFileNameForImport(model.BulkFileUploadModel.BulkFileType, fileName));
var connectionInformation = await GetConnectionInformationProvider();
var config = await _odsSecretConfigurationProvider.GetSecretConfiguration(_instanceContext.Id);
if (config == null)
{
throw new InvalidOperationException("ODS secret configuration can not be null.");
}
var schemaBasePath = HostingEnvironment.MapPath(ConfigurationManager.AppSettings["XsdFolder"]);
var standardVersion = _inferOdsApiVersion.EdFiStandardVersion(connectionInformation.ApiServerUrl);
var jobContext = new BulkUploadJobContext
{
DataDirectoryFullPath = uploadedFiles.Directory,
Environment = CloudOdsEnvironment.Production.Value,
OdsInstanceId = _instanceContext.Id,
OdsInstanceName = _instanceContext.Name,
ApiBaseUrl = connectionInformation.ApiBaseUrl,
OauthUrl = connectionInformation.OAuthUrl,
MetadataUrl = connectionInformation.MetadataUrl,
DependenciesUrl = connectionInformation.DependenciesUrl,
ClientKey = config.BulkUploadCredential?.ApiKey ?? string.Empty,
ClientSecret = config.BulkUploadCredential?.ApiSecret ?? string.Empty,
SchemaPath = $"{schemaBasePath}\\{standardVersion}"
};
if (!_bulkUploadJob.IsJobRunning())
{
_bulkUploadJob.EnqueueJob(jobContext);
}
var updatedValue = model;
updatedValue.BulkFileUploadModel.IsJobRunning = _bulkUploadJob.IsJobRunning();
updatedValue.BulkFileUploadModel.IsSameOdsInstance = _bulkUploadJob.IsSameOdsInstance(_instanceContext.Id, typeof(BulkUploadJobContext));
return(PartialView("_SignalRStatus_BulkLoad", updatedValue));
}
public async Task <ActionResult> BulkFileUpload(BulkFileUploadModel model)
{
var bulkFiles = model.BulkFiles.Where(file => file != null && file.Length > 0).ToArray();
if (!bulkFiles.Any())
{
throw new Exception("The given file is empty. Please upload a file compatible with the Ed-Fi Data Standard.");
}
if (bulkFiles.Sum(f => f.Length) > BulkFileUploadModel.MaxFileSize)
{
throw new Exception($"Upload exceeds maximum limit of {BulkFileUploadModel.MaxFileSize} bytes");
}
if (bulkFiles.Length > 1)
{
throw new Exception("Currently, the bulk import process only supports a single file at a time");
}
InterchangeFileType bulkFileUploadType = null;
if (model.BulkFileType != null)
{
bulkFileUploadType = InterchangeFileType.FromInt32(model.BulkFileType.Value);
}
var uploadedFiles = _fileUploadHandler.SaveFilesToUploadDirectory(bulkFiles,
fileName => InterchangeFileHelpers.BuildFileNameForImport(bulkFileUploadType, fileName), _webHostEnvironment);
var connectionInformation = await GetConnectionInformationProvider();
var config = await _odsSecretConfigurationProvider.GetSecretConfiguration(_instanceContext.Id);
if (config == null)
{
throw new InvalidOperationException("ODS secret configuration can not be null.");
}
if (model != null)
{
model.ApiKey = config.BulkUploadCredential?.ApiKey;
var validationResult = await _bulkLoadValidator.ValidateAsync(model);
if (!validationResult.IsValid)
{
var errorMessage = string.Join(",", validationResult.Errors.Select(x => x.ErrorMessage));
Response.StatusCode = (int)HttpStatusCode.BadRequest;
return(Json(new { Result = new { Errors = new[] { new { ErrorMessage = errorMessage } } } }));
}
}
var schemaBasePath = Path.Combine(_webHostEnvironment.ContentRootPath, _appSettings.XsdFolder);
var standardVersion = await _inferOdsApiVersion.EdFiStandardVersion(connectionInformation.ApiServerUrl);
var odsApiVersion = await _inferOdsApiVersion.Version(connectionInformation.ApiServerUrl);
const int IdealSimultaneousRequests = 20;
const int PessimisticSimultaneousRequests = 1;
var maxSimultaneousRequests = odsApiVersion.StartsWith("3.")
? PessimisticSimultaneousRequests
: IdealSimultaneousRequests;
var jobContext = new BulkUploadJobContext
{
ApiServerUrl = connectionInformation.ApiServerUrl,
DataDirectoryFullPath = uploadedFiles.Directory,
OdsInstanceId = _instanceContext.Id,
OdsInstanceName = _instanceContext.Name,
ApiBaseUrl = connectionInformation.ApiBaseUrl,
OauthUrl = connectionInformation.OAuthUrl,
MetadataUrl = connectionInformation.MetadataUrl,
DependenciesUrl = connectionInformation.DependenciesUrl,
ClientKey = config.BulkUploadCredential?.ApiKey ?? string.Empty,
ClientSecret = config.BulkUploadCredential?.ApiSecret ?? string.Empty,
SchemaPath = Path.Combine(schemaBasePath, standardVersion),
MaxSimultaneousRequests = maxSimultaneousRequests
};
if (!_bulkUploadJob.IsJobRunning())
{
_bulkUploadJob.EnqueueJob(jobContext);
}
var updatedValue = model;
updatedValue.IsJobRunning = _bulkUploadJob.IsJobRunning();
updatedValue.IsSameOdsInstance = _bulkUploadJob.IsSameOdsInstance(_instanceContext.Id, typeof(BulkUploadJobContext));
return(PartialView("_SignalRStatus_BulkLoad", updatedValue));
}