public WorkflowStatus GetStatus(BulkUploadJobContext bulkUploadJobContext, BulkUploadWorkflowManager workflowManager) { if (workflowManager == null) { return(new WorkflowStatus { StatusMessage = "Operation not yet started.", Complete = false, CurrentStep = 0, Error = false, TotalSteps = 0 }); } if (workflowManager.Status.Complete && bulkUploadJobContext.ValidationResult != null && !bulkUploadJobContext.ValidationResult.Valid) { var errorMessage = $"XML Validation failed: {string.Join(Environment.NewLine, bulkUploadJobContext.ValidationResult.ValidationErrors)}"; var status = workflowManager.Status; status.Error = true; status.ErrorMessage = errorMessage; return(status); } return(workflowManager.Status); }
public ActionResult BulkFileUpload(BulkFileUploadModel model) { var bulkFiles = model.BulkFiles.Where(file => file != null && file.ContentLength > 0).ToArray(); if (!bulkFiles.Any()) { return(new HttpStatusCodeResult(HttpStatusCode.NoContent)); } if (bulkFiles.Sum(f => f.ContentLength) > BulkFileUploadModel.MaxFileSize) { throw new Exception($"Upload exceeds maximum limit of {BulkFileUploadModel.MaxFileSize} bytes"); } if (bulkFiles.Length > 1) { throw new Exception("Currently, the bulk import process only supports a single file at a time"); } var uploadedFiles = _fileUploadHandler.SaveFilesToUploadDirectory(bulkFiles, fileName => InterchangeFileHelpers.BuildFileNameForImport(model.BulkFileType, fileName)); var jobContext = new BulkUploadJobContext { DataDirectoryFullPath = uploadedFiles.Directory }; _bulkUploadJob.EnqueueJob(jobContext); return(new HttpStatusCodeResult(HttpStatusCode.Accepted)); }
public async Task <ActionResult> BulkFileUpload(OdsInstanceSettingsModel model) { var bulkFiles = model.BulkFileUploadModel.BulkFiles.Where(file => file != null && file.ContentLength > 0).ToArray(); if (!bulkFiles.Any()) { return(new HttpStatusCodeResult(HttpStatusCode.NoContent)); } if (bulkFiles.Sum(f => f.ContentLength) > BulkFileUploadModel.MaxFileSize) { throw new Exception($"Upload exceeds maximum limit of {BulkFileUploadModel.MaxFileSize} bytes"); } if (bulkFiles.Length > 1) { throw new Exception("Currently, the bulk import process only supports a single file at a time"); } var uploadedFiles = _fileUploadHandler.SaveFilesToUploadDirectory(bulkFiles, fileName => InterchangeFileHelpers.BuildFileNameForImport(model.BulkFileUploadModel.BulkFileType, fileName)); var connectionInformation = await GetConnectionInformationProvider(); var config = await _odsSecretConfigurationProvider.GetSecretConfiguration(_instanceContext.Id); if (config == null) { throw new InvalidOperationException("ODS secret configuration can not be null."); } var schemaBasePath = HostingEnvironment.MapPath(ConfigurationManager.AppSettings["XsdFolder"]); var standardVersion = _inferOdsApiVersion.EdFiStandardVersion(connectionInformation.ApiServerUrl); var jobContext = new BulkUploadJobContext { DataDirectoryFullPath = uploadedFiles.Directory, Environment = CloudOdsEnvironment.Production.Value, OdsInstanceId = _instanceContext.Id, OdsInstanceName = _instanceContext.Name, ApiBaseUrl = connectionInformation.ApiBaseUrl, OauthUrl = connectionInformation.OAuthUrl, MetadataUrl = connectionInformation.MetadataUrl, DependenciesUrl = connectionInformation.DependenciesUrl, ClientKey = config.BulkUploadCredential?.ApiKey ?? string.Empty, ClientSecret = config.BulkUploadCredential?.ApiSecret ?? string.Empty, SchemaPath = $"{schemaBasePath}\\{standardVersion}" }; if (!_bulkUploadJob.IsJobRunning()) { _bulkUploadJob.EnqueueJob(jobContext); } var updatedValue = model; updatedValue.BulkFileUploadModel.IsJobRunning = _bulkUploadJob.IsJobRunning(); updatedValue.BulkFileUploadModel.IsSameOdsInstance = _bulkUploadJob.IsSameOdsInstance(_instanceContext.Id, typeof(BulkUploadJobContext)); return(PartialView("_SignalRStatus_BulkLoad", updatedValue)); }
private BulkUploadWorkflowManager SetupWorkflowManager(BulkUploadJobContext bulkUploadJobContext, FileImportService fileImportService) { var workflowManager = new BulkUploadWorkflowManager(bulkUploadJobContext); workflowManager.JobStarted += () => OnStatusUpdated(GetStatus(bulkUploadJobContext, workflowManager)); workflowManager.JobCompleted += () => OnStatusUpdated(GetStatus(bulkUploadJobContext, workflowManager)); workflowManager.StepStarted += () => OnStatusUpdated(GetStatus(bulkUploadJobContext, workflowManager)); workflowManager.StepCompleted += () => OnStatusUpdated(GetStatus(bulkUploadJobContext, workflowManager)); workflowManager .StartWith(new BulkUploadWorkflowStep { StatusMessage = "Authenticating", FailureMessage = "Authentication failed. Please ensure your credentials are valid", RollBackAction = CleanUp, ExecuteAction = context => Authenticate(fileImportService), RollbackFailureMessage = "An error occured while authenticating", RollbackStatusMessage = "An error occured while authenticating" }) .ContinueWith(new BulkUploadWorkflowStep { StatusMessage = "Validating XML", FailureMessage = "An error occured validating XML file content.", RollBackAction = CleanUp, ExecuteAction = context => ValidateBulkDataXml(context, fileImportService), RollbackFailureMessage = "An error occured validating XML file content", RollbackStatusMessage = "An error occured validating XML file content" }) .ContinueWith(new BulkUploadWorkflowStep { StatusMessage = "Importing data", FailureMessage = "Data import failed. {0}", RollBackAction = CleanUp, ExecuteAction = context => RunImport(fileImportService), RollbackFailureMessage = "Data import failed", RollbackStatusMessage = "Data import failed" }) .ContinueWith(new BulkUploadWorkflowStep { StatusMessage = "Cleaning up temporary files", FailureMessage = "Import completed, but there was an error removing the temporary files. {0}", RollBackAction = context => { }, ExecuteAction = CleanUp, RollbackFailureMessage = "", RollbackStatusMessage = "" }).ContinueWith(new BulkUploadWorkflowStep { StatusMessage = "Import completed, please check Admin App logs to see the details of your import.", FailureMessage = "", RollBackAction = context => { }, ExecuteAction = context => { }, RollbackFailureMessage = "", RollbackStatusMessage = "" }); return(workflowManager); }
public void CleanUp(BulkUploadJobContext jobContext) { var uploadFolder = jobContext?.DataDirectoryFullPath; if (!string.IsNullOrEmpty(uploadFolder) && Directory.Exists(uploadFolder)) { Directory.Delete(uploadFolder, true); } }
private static FileImportConfiguration GetFileImportConfig(BulkUploadJobContext bulkUploadJobContext, string workingFolderPath) { return(new FileImportConfiguration( bulkUploadJobContext.DataDirectoryFullPath, workingFolderPath, bulkUploadJobContext.SchoolYear, apiBaseUrl: bulkUploadJobContext.ApiBaseUrl, clientKey: bulkUploadJobContext.ClientKey, clientSecret: bulkUploadJobContext.ClientSecret, oauthUrl: bulkUploadJobContext.OauthUrl, metadataUrl: bulkUploadJobContext.MetadataUrl, dependenciesUrl: bulkUploadJobContext.DependenciesUrl, schemaPath: bulkUploadJobContext.SchemaPath, maxSimultaneousRequests: bulkUploadJobContext.MaxSimultaneousRequests)); }
private void ValidateBulkDataXml(BulkUploadJobContext jobContext, FileImportService fileImportService) { try { jobContext.ValidationResult = fileImportService.Validate(); if (jobContext.ValidationResult != null && !jobContext.ValidationResult.Valid) { throw new Exception("Validation failed"); } } catch (Exception ex) { throw new Exception($"An error occurred during file validation: {ex.Message}"); } }
public WorkflowResult Execute(BulkUploadJobContext bulkUploadJobContext) { try { var globalBulkUploadFolder = CloudOdsAdminAppSettings.AppSettings.BulkUploadHashCache; var instanceBulkUploadFolder = $"{globalBulkUploadFolder}\\{bulkUploadJobContext.OdsInstanceId}"; var workingFolderPath = _fileUploadHandler.GetWorkingDirectory(instanceBulkUploadFolder); var fileImportConfig = new FileImportConfiguration().SetConfiguration( bulkUploadJobContext, workingFolderPath); var fileImportService = new FileImportService( fileImportConfig, bulkUploadJobContext.OdsInstanceName, _inferOdsApiVersion); var workflowManager = SetupWorkflowManager(bulkUploadJobContext, fileImportService); return(workflowManager.Execute()); } catch (Exception ex) { var errorMessage = $"An error occured while initializing the file importer: {ex.Message}"; var status = new WorkflowStatus { Complete = true, CurrentStep = 0, TotalSteps = 0, Error = true, ErrorMessage = errorMessage, }; StatusUpdated?.Invoke(status); return(new WorkflowResult { Error = true, ErrorMessage = errorMessage, TotalSteps = 0, StepsCompletedSuccessfully = 0, }); } }
public IConfiguration SetConfiguration(BulkUploadJobContext bulkUploadJobContext, string workingFolderPath) { Require(bulkUploadJobContext.ApiBaseUrl); Require(bulkUploadJobContext.ClientKey); Require(bulkUploadJobContext.ClientSecret); Require(bulkUploadJobContext.OauthUrl); Require(bulkUploadJobContext.MetadataUrl); Require(bulkUploadJobContext.DependenciesUrl); var builder = new ConfigurationBuilder(); var configuration = builder.Add(new BulkUploadConfigurationSource()) .Build(); configuration["OdsApi:Url"] = bulkUploadJobContext.ApiServerUrl; configuration["Folders:Data"] = bulkUploadJobContext.DataDirectoryFullPath; configuration["Folders:Working"] = workingFolderPath; configuration["OdsApi:ApiUrl"] = bulkUploadJobContext.ApiBaseUrl + "/"; configuration["OdsApi:Key"] = bulkUploadJobContext.ClientKey; configuration["OdsApi:Secret"] = bulkUploadJobContext.ClientSecret; configuration["OdsApi:OAuthUrl"] = bulkUploadJobContext.OauthUrl; configuration["OdsApi:MetadataUrl"] = bulkUploadJobContext.MetadataUrl; configuration["Folders:Xsd"] = bulkUploadJobContext.SchemaPath; configuration["Folders:Interchange"] = bulkUploadJobContext.SchemaPath; configuration["Concurrency:MaxRetries"] = "3"; configuration["OdsApi:SchoolYear"] = bulkUploadJobContext.SchoolYear.ToString(); configuration["Concurrency:ConnectionLimit"] = bulkUploadJobContext.MaxSimultaneousRequests.ToString(); configuration["Concurrency:MaxSimultaneousApiRequests"] = bulkUploadJobContext.MaxSimultaneousRequests.ToString(); configuration["Concurrency:TaskCapacity"] = bulkUploadJobContext.MaxSimultaneousRequests.ToString(); configuration["OdsApi:DependenciesUrl"] = bulkUploadJobContext.DependenciesUrl; configuration["ValidateSchema"] = "false"; configuration["ForceMetadata"] = "true"; return(configuration); }
private void ValidateBulkDataXml(BulkUploadJobContext jobContext, FileImportService fileImportService) { jobContext.ValidationResult = fileImportService.Validate(); }
public async Task <ActionResult> BulkFileUpload(BulkFileUploadModel model) { var bulkFiles = model.BulkFiles.Where(file => file != null && file.Length > 0).ToArray(); if (!bulkFiles.Any()) { throw new Exception("The given file is empty. Please upload a file compatible with the Ed-Fi Data Standard."); } if (bulkFiles.Sum(f => f.Length) > BulkFileUploadModel.MaxFileSize) { throw new Exception($"Upload exceeds maximum limit of {BulkFileUploadModel.MaxFileSize} bytes"); } if (bulkFiles.Length > 1) { throw new Exception("Currently, the bulk import process only supports a single file at a time"); } InterchangeFileType bulkFileUploadType = null; if (model.BulkFileType != null) { bulkFileUploadType = InterchangeFileType.FromInt32(model.BulkFileType.Value); } var uploadedFiles = _fileUploadHandler.SaveFilesToUploadDirectory(bulkFiles, fileName => InterchangeFileHelpers.BuildFileNameForImport(bulkFileUploadType, fileName), _webHostEnvironment); var connectionInformation = await GetConnectionInformationProvider(); var config = await _odsSecretConfigurationProvider.GetSecretConfiguration(_instanceContext.Id); if (config == null) { throw new InvalidOperationException("ODS secret configuration can not be null."); } if (model != null) { model.ApiKey = config.BulkUploadCredential?.ApiKey; var validationResult = await _bulkLoadValidator.ValidateAsync(model); if (!validationResult.IsValid) { var errorMessage = string.Join(",", validationResult.Errors.Select(x => x.ErrorMessage)); Response.StatusCode = (int)HttpStatusCode.BadRequest; return(Json(new { Result = new { Errors = new[] { new { ErrorMessage = errorMessage } } } })); } } var schemaBasePath = Path.Combine(_webHostEnvironment.ContentRootPath, _appSettings.XsdFolder); var standardVersion = await _inferOdsApiVersion.EdFiStandardVersion(connectionInformation.ApiServerUrl); var odsApiVersion = await _inferOdsApiVersion.Version(connectionInformation.ApiServerUrl); const int IdealSimultaneousRequests = 20; const int PessimisticSimultaneousRequests = 1; var maxSimultaneousRequests = odsApiVersion.StartsWith("3.") ? PessimisticSimultaneousRequests : IdealSimultaneousRequests; var jobContext = new BulkUploadJobContext { ApiServerUrl = connectionInformation.ApiServerUrl, DataDirectoryFullPath = uploadedFiles.Directory, OdsInstanceId = _instanceContext.Id, OdsInstanceName = _instanceContext.Name, ApiBaseUrl = connectionInformation.ApiBaseUrl, OauthUrl = connectionInformation.OAuthUrl, MetadataUrl = connectionInformation.MetadataUrl, DependenciesUrl = connectionInformation.DependenciesUrl, ClientKey = config.BulkUploadCredential?.ApiKey ?? string.Empty, ClientSecret = config.BulkUploadCredential?.ApiSecret ?? string.Empty, SchemaPath = Path.Combine(schemaBasePath, standardVersion), MaxSimultaneousRequests = maxSimultaneousRequests }; if (!_bulkUploadJob.IsJobRunning()) { _bulkUploadJob.EnqueueJob(jobContext); } var updatedValue = model; updatedValue.IsJobRunning = _bulkUploadJob.IsJobRunning(); updatedValue.IsSameOdsInstance = _bulkUploadJob.IsSameOdsInstance(_instanceContext.Id, typeof(BulkUploadJobContext)); return(PartialView("_SignalRStatus_BulkLoad", updatedValue)); }