private async Task <bool> ResolveCoordinateSystemFromRaptor(MigrationJob job) { _log.LogInformation($"{Method.In()} Resolving project {job.Project.ProjectUID} CSIB from Raptor"); var logMessage = $"Failed to fetch coordinate system file '{job.Project.CustomerUID}/{job.Project.ProjectUID}/{job.Project.CoordinateSystemFileName}' from TCC."; _migrationDb.Insert(new MigrationMessage(job.Project.ProjectUID, logMessage), Table.Warnings); _log.LogWarning(logMessage); // Get the the CSIB for the project from Raptor. var csibResponse = await _csibAgent.GetCSIBForProject(job.Project); var csib = csibResponse.CSIB; if (csibResponse.Code != 0) { const string errorMessage = "Failed to resolve CSIB from Raptor"; _migrationDb.SetResolveCSIBMessage(Table.Projects, job.Project.ProjectUID, csib); _migrationDb.Insert(new MigrationMessage(job.Project.ProjectUID, errorMessage), Table.Errors); _log.LogWarning(errorMessage); return(false); } _migrationDb.SetProjectCSIB(Table.Projects, job.Project.ProjectUID, csib); var coordSysInfo = await _csibAgent.GetCoordSysInfoFromCSIB64(job.Project, csib); var dcFileContent = await _csibAgent.GetCalibrationFileForCoordSysId(job.Project, coordSysInfo["coordinateSystem"]["id"].ToString()); var coordSystemFileContent = Encoding.UTF8.GetBytes(dcFileContent); using (var stream = new MemoryStream(coordSystemFileContent)) { if (SaveDCFileToDisk(job, stream)) { return(true); } } _log.LogError("Failed to resolve coordinate system information from Raptor"); return(false); }
public async Task MigrateFilesForAllActiveProjects() { Log.LogInformation($"{Method.Info()} Fetching projects..."); var projects = (await ProjectRepo.GetActiveProjects()).ToList(); Log.LogInformation($"{Method.Info()} Found {projects.Count} projects"); var inputProjects = _appSettings.GetSection("Projects") .Get <string[]>(); var ignoredProjects = _appSettings.GetSection("IgnoredProjects") .Get <string[]>(); _ignoredFiles = _appSettings.GetSection("IgnoredFiles") .Get <string[]>(); // Are we processing only a subset of projects from the appSettings::Projects array? if (inputProjects != null && inputProjects.Any()) { Log.LogInformation($"{Method.Info()} Found {inputProjects.Length} input projects to process."); var tmpProjects = new List <Project>(inputProjects.Length); foreach (var projectUid in inputProjects) { var project = projects.Find(x => x.ProjectUID == projectUid); if (project != null) { Log.LogInformation($"{Method.Info()} Adding {project.ProjectUID}"); tmpProjects.Add(project); } } if (!projects.Any()) { Log.LogInformation($"{Method.Info()} Unable to resolve any projects to process, exiting."); return; } DropTables(); projects = tmpProjects; } else { if (!_resumeMigration) { DropTables(); if (Directory.Exists(_tempFolder)) { Log.LogDebug($"{Method.Info()} Removing temporary files from {_tempFolder}"); Directory.Delete(_tempFolder, recursive: true); } } } if (!_resumeMigration) { MigrationInfoId = _database.Insert(new MigrationInfo()); } var projectCount = Math.Min(projects.Count, _capMigrationCount); var projectTasks = new List <Task <bool> >(projectCount); _database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsTotal = projectCount); var projectsProcessed = 0; var processedProjects = new List <string>(); foreach (var project in projects) { if (ignoredProjects != null && ignoredProjects.Contains(project.ProjectUID)) { Log.LogInformation($"{Method.Info()} Ignoring project {project.ProjectUID}; found in IgnoredProjects list."); continue; } var projectRecord = _database.Find <MigrationProject>(Table.Projects, project.LegacyProjectID); if (projectRecord == null) { Log.LogInformation($"{Method.Info()} Creating new migration record for project {project.ProjectUID}"); _database.Insert(new MigrationProject(project)); } else { Log.LogInformation($"{Method.Info()} Found migration record for project {project.ProjectUID}"); // TODO Check completed=true & eligibleFiles > 0 && uploadedFiles=0; should retry. if (projectRecord.MigrationState == MigrationState.Completed && !_reProcessSkippedFiles) { Log.LogInformation($"{Method.Info()} Skipping project {project.ProjectUID}, marked as COMPLETED"); continue; } if (projectRecord.MigrationState != MigrationState.Completed) { if (!_reProcessFailedProjects) { Log.LogInformation($"{Method.Info()} Not reprocessing {Enum.GetName(typeof(MigrationState), projectRecord.MigrationState)?.ToUpper()} project {project.ProjectUID}"); continue; } } Log.LogInformation($"{Method.Info()} Resuming migration for project {project.ProjectUID}, marked as {Enum.GetName(typeof(MigrationState), projectRecord.MigrationState)?.ToUpper()}"); } var job = new MigrationJob { Project = project, IsRetryAttempt = projectRecord != null }; if (projectsProcessed <= _capMigrationCount) { processedProjects.Add(job.Project.ProjectUID); projectTasks.Add(MigrateProject(job)); } if (projectTasks.Count <= THROTTLE_ASYNC_PROJECT_JOBS && projectsProcessed < _capMigrationCount - 1) { continue; } var completed = await Task.WhenAny(projectTasks); projectTasks.Remove(completed); _database.IncrementProjectMigrationCounter(project); projectsProcessed += 1; Log.LogInformation("Migration Progress:"); Log.LogInformation($" Processed: {projectsProcessed}"); Log.LogInformation($" In Flight: {projectTasks.Count}"); Log.LogInformation($" Remaining: {projectCount - projectsProcessed}"); if (projectsProcessed >= _capMigrationCount) { Log.LogInformation($"{Method.Info()} Reached maxium number of projects to process, exiting."); break; } } await Task.WhenAll(projectTasks); // DIAGNOSTIC RUNTIME SWITCH if (_saveFailedProjects) { // Create a recovery file of project uids for re processing var failedProjectsLog = Path.Combine(_tempFolder, $"FailedProjects{DateTime.Now.Date.ToShortDateString().Replace('/', '-')}_{DateTime.Now.Hour}{DateTime.Now.Minute}{DateTime.Now.Second}.log"); var completedProjectsLog = Path.Combine(_tempFolder, $"Completed{DateTime.Now.Date.ToShortDateString().Replace('/', '-')}_{DateTime.Now.Hour}{DateTime.Now.Minute}{DateTime.Now.Second}.log"); if (!Directory.Exists(_tempFolder)) { Directory.CreateDirectory(_tempFolder); } var allProjects = _database.GetTable <MigrationProject>(Table.Projects).ToList(); using (TextWriter streamWriterFailed = new StreamWriter(failedProjectsLog)) using (TextWriter streamWriterCompleted = new StreamWriter(completedProjectsLog)) { foreach (var project in processedProjects) { var migrationProject = allProjects.FirstOrDefault(x => x.ProjectUid == project); if (migrationProject == null) { continue; } if (migrationProject.MigrationState == MigrationState.Completed) { streamWriterCompleted.WriteLine($"{migrationProject.ProjectUid}"); continue; } var message = string.IsNullOrEmpty(migrationProject.MigrationStateMessage) ? null : $" // {migrationProject.MigrationStateMessage}"; streamWriterFailed.WriteLine($"{migrationProject.ProjectUid}{message}"); } } } // Set the final summary figures. var completedCount = _database.Find <MigrationProject>(Table.Projects, x => x.MigrationState == MigrationState.Completed) .Count(); _database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsSuccessful = completedCount); var failedCount = _database.Find <MigrationProject>(Table.Projects, x => x.MigrationState == MigrationState.Failed) .Count(); _database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsFailed = failedCount); Log.LogInformation("Migration processing completed."); }