public WorkflowStatus GetStatus(BulkUploadJobContext bulkUploadJobContext, BulkUploadWorkflowManager workflowManager)
{
if (workflowManager == null)
{
return(new WorkflowStatus
{
StatusMessage = "Operation not yet started.",
Complete = false,
CurrentStep = 0,
Error = false,
TotalSteps = 0
});
}
if (workflowManager.Status.Complete && bulkUploadJobContext.ValidationResult != null && !bulkUploadJobContext.ValidationResult.Valid)
{
var errorMessage = $"XML Validation failed: {string.Join(Environment.NewLine, bulkUploadJobContext.ValidationResult.ValidationErrors)}";
var status = workflowManager.Status;
status.Error = true;
status.ErrorMessage = errorMessage;
return(status);
}
return(workflowManager.Status);
}
public ActionResult BulkFileUpload(BulkFileUploadModel model)
{
var bulkFiles = model.BulkFiles.Where(file => file != null && file.ContentLength > 0).ToArray();
if (!bulkFiles.Any())
{
return(new HttpStatusCodeResult(HttpStatusCode.NoContent));
}
if (bulkFiles.Sum(f => f.ContentLength) > BulkFileUploadModel.MaxFileSize)
{
throw new Exception($"Upload exceeds maximum limit of {BulkFileUploadModel.MaxFileSize} bytes");
}
if (bulkFiles.Length > 1)
{
throw new Exception("Currently, the bulk import process only supports a single file at a time");
}
var uploadedFiles = _fileUploadHandler.SaveFilesToUploadDirectory(bulkFiles, fileName => InterchangeFileHelpers.BuildFileNameForImport(model.BulkFileType, fileName));
var jobContext = new BulkUploadJobContext
{
DataDirectoryFullPath = uploadedFiles.Directory
};
_bulkUploadJob.EnqueueJob(jobContext);
return(new HttpStatusCodeResult(HttpStatusCode.Accepted));
}
public async Task <ActionResult> BulkFileUpload(OdsInstanceSettingsModel model)
{
var bulkFiles = model.BulkFileUploadModel.BulkFiles.Where(file => file != null && file.ContentLength > 0).ToArray();
if (!bulkFiles.Any())
{
return(new HttpStatusCodeResult(HttpStatusCode.NoContent));
}
if (bulkFiles.Sum(f => f.ContentLength) > BulkFileUploadModel.MaxFileSize)
{
throw new Exception($"Upload exceeds maximum limit of {BulkFileUploadModel.MaxFileSize} bytes");
}
if (bulkFiles.Length > 1)
{
throw new Exception("Currently, the bulk import process only supports a single file at a time");
}
var uploadedFiles = _fileUploadHandler.SaveFilesToUploadDirectory(bulkFiles,
fileName => InterchangeFileHelpers.BuildFileNameForImport(model.BulkFileUploadModel.BulkFileType, fileName));
var connectionInformation = await GetConnectionInformationProvider();
var config = await _odsSecretConfigurationProvider.GetSecretConfiguration(_instanceContext.Id);
if (config == null)
{
throw new InvalidOperationException("ODS secret configuration can not be null.");
}
var schemaBasePath = HostingEnvironment.MapPath(ConfigurationManager.AppSettings["XsdFolder"]);
var standardVersion = _inferOdsApiVersion.EdFiStandardVersion(connectionInformation.ApiServerUrl);
var jobContext = new BulkUploadJobContext
{
DataDirectoryFullPath = uploadedFiles.Directory,
Environment = CloudOdsEnvironment.Production.Value,
OdsInstanceId = _instanceContext.Id,
OdsInstanceName = _instanceContext.Name,
ApiBaseUrl = connectionInformation.ApiBaseUrl,
OauthUrl = connectionInformation.OAuthUrl,
MetadataUrl = connectionInformation.MetadataUrl,
DependenciesUrl = connectionInformation.DependenciesUrl,
ClientKey = config.BulkUploadCredential?.ApiKey ?? string.Empty,
ClientSecret = config.BulkUploadCredential?.ApiSecret ?? string.Empty,
SchemaPath = $"{schemaBasePath}\\{standardVersion}"
};
if (!_bulkUploadJob.IsJobRunning())
{
_bulkUploadJob.EnqueueJob(jobContext);
}
var updatedValue = model;
updatedValue.BulkFileUploadModel.IsJobRunning = _bulkUploadJob.IsJobRunning();
updatedValue.BulkFileUploadModel.IsSameOdsInstance = _bulkUploadJob.IsSameOdsInstance(_instanceContext.Id, typeof(BulkUploadJobContext));
return(PartialView("_SignalRStatus_BulkLoad", updatedValue));
}
private BulkUploadWorkflowManager SetupWorkflowManager(BulkUploadJobContext bulkUploadJobContext, FileImportService fileImportService)
{
var workflowManager = new BulkUploadWorkflowManager(bulkUploadJobContext);
workflowManager.JobStarted += () => OnStatusUpdated(GetStatus(bulkUploadJobContext, workflowManager));
workflowManager.JobCompleted += () => OnStatusUpdated(GetStatus(bulkUploadJobContext, workflowManager));
workflowManager.StepStarted += () => OnStatusUpdated(GetStatus(bulkUploadJobContext, workflowManager));
workflowManager.StepCompleted += () => OnStatusUpdated(GetStatus(bulkUploadJobContext, workflowManager));
workflowManager
.StartWith(new BulkUploadWorkflowStep
{
StatusMessage = "Authenticating",
FailureMessage = "Authentication failed. Please ensure your credentials are valid",
RollBackAction = CleanUp,
ExecuteAction = context => Authenticate(fileImportService),
RollbackFailureMessage = "An error occured while authenticating",
RollbackStatusMessage = "An error occured while authenticating"
})
.ContinueWith(new BulkUploadWorkflowStep
{
StatusMessage = "Validating XML",
FailureMessage = "An error occured validating XML file content.",
RollBackAction = CleanUp,
ExecuteAction = context => ValidateBulkDataXml(context, fileImportService),
RollbackFailureMessage = "An error occured validating XML file content",
RollbackStatusMessage = "An error occured validating XML file content"
})
.ContinueWith(new BulkUploadWorkflowStep
{
StatusMessage = "Importing data",
FailureMessage = "Data import failed. {0}",
RollBackAction = CleanUp,
ExecuteAction = context => RunImport(fileImportService),
RollbackFailureMessage = "Data import failed",
RollbackStatusMessage = "Data import failed"
})
.ContinueWith(new BulkUploadWorkflowStep
{
StatusMessage = "Cleaning up temporary files",
FailureMessage = "Import completed, but there was an error removing the temporary files. {0}",
RollBackAction = context => { },
ExecuteAction = CleanUp,
RollbackFailureMessage = "",
RollbackStatusMessage = ""
}).ContinueWith(new BulkUploadWorkflowStep
{
StatusMessage = "Import completed, please check Admin App logs to see the details of your import.",
FailureMessage = "",
RollBackAction = context => { },
ExecuteAction = context => { },
RollbackFailureMessage = "",
RollbackStatusMessage = ""
});
return(workflowManager);
}
public void CleanUp(BulkUploadJobContext jobContext)
{
var uploadFolder = jobContext?.DataDirectoryFullPath;
if (!string.IsNullOrEmpty(uploadFolder) && Directory.Exists(uploadFolder))
{
Directory.Delete(uploadFolder, true);
}
}
private static FileImportConfiguration GetFileImportConfig(BulkUploadJobContext bulkUploadJobContext, string workingFolderPath)
{
return(new FileImportConfiguration(
bulkUploadJobContext.DataDirectoryFullPath,
workingFolderPath,
bulkUploadJobContext.SchoolYear,
apiBaseUrl: bulkUploadJobContext.ApiBaseUrl,
clientKey: bulkUploadJobContext.ClientKey,
clientSecret: bulkUploadJobContext.ClientSecret,
oauthUrl: bulkUploadJobContext.OauthUrl,
metadataUrl: bulkUploadJobContext.MetadataUrl,
dependenciesUrl: bulkUploadJobContext.DependenciesUrl,
schemaPath: bulkUploadJobContext.SchemaPath,
maxSimultaneousRequests: bulkUploadJobContext.MaxSimultaneousRequests));
}
private void ValidateBulkDataXml(BulkUploadJobContext jobContext, FileImportService fileImportService)
{
try
{
jobContext.ValidationResult = fileImportService.Validate();
if (jobContext.ValidationResult != null && !jobContext.ValidationResult.Valid)
{
throw new Exception("Validation failed");
}
}
catch (Exception ex)
{
throw new Exception($"An error occurred during file validation: {ex.Message}");
}
}
public WorkflowResult Execute(BulkUploadJobContext bulkUploadJobContext)
{
try
{
var globalBulkUploadFolder = CloudOdsAdminAppSettings.AppSettings.BulkUploadHashCache;
var instanceBulkUploadFolder =
$"{globalBulkUploadFolder}\\{bulkUploadJobContext.OdsInstanceId}";
var workingFolderPath =
_fileUploadHandler.GetWorkingDirectory(instanceBulkUploadFolder);
var fileImportConfig = new FileImportConfiguration().SetConfiguration(
bulkUploadJobContext, workingFolderPath);
var fileImportService = new FileImportService(
fileImportConfig, bulkUploadJobContext.OdsInstanceName, _inferOdsApiVersion);
var workflowManager = SetupWorkflowManager(bulkUploadJobContext, fileImportService);
return(workflowManager.Execute());
}
catch (Exception ex)
{
var errorMessage = $"An error occured while initializing the file importer: {ex.Message}";
var status = new WorkflowStatus
{
Complete = true,
CurrentStep = 0,
TotalSteps = 0,
Error = true,
ErrorMessage = errorMessage,
};
StatusUpdated?.Invoke(status);
return(new WorkflowResult
{
Error = true,
ErrorMessage = errorMessage,
TotalSteps = 0,
StepsCompletedSuccessfully = 0,
});
}
}
public IConfiguration SetConfiguration(BulkUploadJobContext bulkUploadJobContext, string workingFolderPath)
{
Require(bulkUploadJobContext.ApiBaseUrl);
Require(bulkUploadJobContext.ClientKey);
Require(bulkUploadJobContext.ClientSecret);
Require(bulkUploadJobContext.OauthUrl);
Require(bulkUploadJobContext.MetadataUrl);
Require(bulkUploadJobContext.DependenciesUrl);
var builder = new ConfigurationBuilder();
var configuration =
builder.Add(new BulkUploadConfigurationSource())
.Build();
configuration["OdsApi:Url"] = bulkUploadJobContext.ApiServerUrl;
configuration["Folders:Data"] = bulkUploadJobContext.DataDirectoryFullPath;
configuration["Folders:Working"] = workingFolderPath;
configuration["OdsApi:ApiUrl"] = bulkUploadJobContext.ApiBaseUrl + "/";
configuration["OdsApi:Key"] = bulkUploadJobContext.ClientKey;
configuration["OdsApi:Secret"] = bulkUploadJobContext.ClientSecret;
configuration["OdsApi:OAuthUrl"] = bulkUploadJobContext.OauthUrl;
configuration["OdsApi:MetadataUrl"] = bulkUploadJobContext.MetadataUrl;
configuration["Folders:Xsd"] = bulkUploadJobContext.SchemaPath;
configuration["Folders:Interchange"] = bulkUploadJobContext.SchemaPath;
configuration["Concurrency:MaxRetries"] = "3";
configuration["OdsApi:SchoolYear"] = bulkUploadJobContext.SchoolYear.ToString();
configuration["Concurrency:ConnectionLimit"] =
bulkUploadJobContext.MaxSimultaneousRequests.ToString();
configuration["Concurrency:MaxSimultaneousApiRequests"] =
bulkUploadJobContext.MaxSimultaneousRequests.ToString();
configuration["Concurrency:TaskCapacity"] =
bulkUploadJobContext.MaxSimultaneousRequests.ToString();
configuration["OdsApi:DependenciesUrl"] = bulkUploadJobContext.DependenciesUrl;
configuration["ValidateSchema"] = "false";
configuration["ForceMetadata"] = "true";
return(configuration);
}
private void ValidateBulkDataXml(BulkUploadJobContext jobContext, FileImportService fileImportService)
{
jobContext.ValidationResult = fileImportService.Validate();
}
public async Task <ActionResult> BulkFileUpload(BulkFileUploadModel model)
{
var bulkFiles = model.BulkFiles.Where(file => file != null && file.Length > 0).ToArray();
if (!bulkFiles.Any())
{
throw new Exception("The given file is empty. Please upload a file compatible with the Ed-Fi Data Standard.");
}
if (bulkFiles.Sum(f => f.Length) > BulkFileUploadModel.MaxFileSize)
{
throw new Exception($"Upload exceeds maximum limit of {BulkFileUploadModel.MaxFileSize} bytes");
}
if (bulkFiles.Length > 1)
{
throw new Exception("Currently, the bulk import process only supports a single file at a time");
}
InterchangeFileType bulkFileUploadType = null;
if (model.BulkFileType != null)
{
bulkFileUploadType = InterchangeFileType.FromInt32(model.BulkFileType.Value);
}
var uploadedFiles = _fileUploadHandler.SaveFilesToUploadDirectory(bulkFiles,
fileName => InterchangeFileHelpers.BuildFileNameForImport(bulkFileUploadType, fileName), _webHostEnvironment);
var connectionInformation = await GetConnectionInformationProvider();
var config = await _odsSecretConfigurationProvider.GetSecretConfiguration(_instanceContext.Id);
if (config == null)
{
throw new InvalidOperationException("ODS secret configuration can not be null.");
}
if (model != null)
{
model.ApiKey = config.BulkUploadCredential?.ApiKey;
var validationResult = await _bulkLoadValidator.ValidateAsync(model);
if (!validationResult.IsValid)
{
var errorMessage = string.Join(",", validationResult.Errors.Select(x => x.ErrorMessage));
Response.StatusCode = (int)HttpStatusCode.BadRequest;
return(Json(new { Result = new { Errors = new[] { new { ErrorMessage = errorMessage } } } }));
}
}
var schemaBasePath = Path.Combine(_webHostEnvironment.ContentRootPath, _appSettings.XsdFolder);
var standardVersion = await _inferOdsApiVersion.EdFiStandardVersion(connectionInformation.ApiServerUrl);
var odsApiVersion = await _inferOdsApiVersion.Version(connectionInformation.ApiServerUrl);
const int IdealSimultaneousRequests = 20;
const int PessimisticSimultaneousRequests = 1;
var maxSimultaneousRequests = odsApiVersion.StartsWith("3.")
? PessimisticSimultaneousRequests
: IdealSimultaneousRequests;
var jobContext = new BulkUploadJobContext
{
ApiServerUrl = connectionInformation.ApiServerUrl,
DataDirectoryFullPath = uploadedFiles.Directory,
OdsInstanceId = _instanceContext.Id,
OdsInstanceName = _instanceContext.Name,
ApiBaseUrl = connectionInformation.ApiBaseUrl,
OauthUrl = connectionInformation.OAuthUrl,
MetadataUrl = connectionInformation.MetadataUrl,
DependenciesUrl = connectionInformation.DependenciesUrl,
ClientKey = config.BulkUploadCredential?.ApiKey ?? string.Empty,
ClientSecret = config.BulkUploadCredential?.ApiSecret ?? string.Empty,
SchemaPath = Path.Combine(schemaBasePath, standardVersion),
MaxSimultaneousRequests = maxSimultaneousRequests
};
if (!_bulkUploadJob.IsJobRunning())
{
_bulkUploadJob.EnqueueJob(jobContext);
}
var updatedValue = model;
updatedValue.IsJobRunning = _bulkUploadJob.IsJobRunning();
updatedValue.IsSameOdsInstance = _bulkUploadJob.IsSameOdsInstance(_instanceContext.Id, typeof(BulkUploadJobContext));
return(PartialView("_SignalRStatus_BulkLoad", updatedValue));
}
