private async Task <bool> ResolveCoordinateSystemFromRaptor(MigrationJob job)
{
_log.LogInformation($"{Method.In()} Resolving project {job.Project.ProjectUID} CSIB from Raptor");
var logMessage = $"Failed to fetch coordinate system file '{job.Project.CustomerUID}/{job.Project.ProjectUID}/{job.Project.CoordinateSystemFileName}' from TCC.";
_migrationDb.Insert(new MigrationMessage(job.Project.ProjectUID, logMessage), Table.Warnings);
_log.LogWarning(logMessage);
// Get the the CSIB for the project from Raptor.
var csibResponse = await _csibAgent.GetCSIBForProject(job.Project);
var csib = csibResponse.CSIB;
if (csibResponse.Code != 0)
{
const string errorMessage = "Failed to resolve CSIB from Raptor";
_migrationDb.SetResolveCSIBMessage(Table.Projects, job.Project.ProjectUID, csib);
_migrationDb.Insert(new MigrationMessage(job.Project.ProjectUID, errorMessage), Table.Errors);
_log.LogWarning(errorMessage);
return(false);
}
_migrationDb.SetProjectCSIB(Table.Projects, job.Project.ProjectUID, csib);
var coordSysInfo = await _csibAgent.GetCoordSysInfoFromCSIB64(job.Project, csib);
var dcFileContent = await _csibAgent.GetCalibrationFileForCoordSysId(job.Project, coordSysInfo["coordinateSystem"]["id"].ToString());
var coordSystemFileContent = Encoding.UTF8.GetBytes(dcFileContent);
using (var stream = new MemoryStream(coordSystemFileContent))
{
if (SaveDCFileToDisk(job, stream))
{
return(true);
}
}
_log.LogError("Failed to resolve coordinate system information from Raptor");
return(false);
}
public async Task MigrateFilesForAllActiveProjects()
{
Log.LogInformation($"{Method.Info()} Fetching projects...");
var projects = (await ProjectRepo.GetActiveProjects()).ToList();
Log.LogInformation($"{Method.Info()} Found {projects.Count} projects");
var inputProjects = _appSettings.GetSection("Projects")
.Get <string[]>();
var ignoredProjects = _appSettings.GetSection("IgnoredProjects")
.Get <string[]>();
_ignoredFiles = _appSettings.GetSection("IgnoredFiles")
.Get <string[]>();
// Are we processing only a subset of projects from the appSettings::Projects array?
if (inputProjects != null && inputProjects.Any())
{
Log.LogInformation($"{Method.Info()} Found {inputProjects.Length} input projects to process.");
var tmpProjects = new List <Project>(inputProjects.Length);
foreach (var projectUid in inputProjects)
{
var project = projects.Find(x => x.ProjectUID == projectUid);
if (project != null)
{
Log.LogInformation($"{Method.Info()} Adding {project.ProjectUID}");
tmpProjects.Add(project);
}
}
if (!projects.Any())
{
Log.LogInformation($"{Method.Info()} Unable to resolve any projects to process, exiting.");
return;
}
DropTables();
projects = tmpProjects;
}
else
{
if (!_resumeMigration)
{
DropTables();
if (Directory.Exists(_tempFolder))
{
Log.LogDebug($"{Method.Info()} Removing temporary files from {_tempFolder}");
Directory.Delete(_tempFolder, recursive: true);
}
}
}
if (!_resumeMigration)
{
MigrationInfoId = _database.Insert(new MigrationInfo());
}
var projectCount = Math.Min(projects.Count, _capMigrationCount);
var projectTasks = new List <Task <bool> >(projectCount);
_database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsTotal = projectCount);
var projectsProcessed = 0;
var processedProjects = new List <string>();
foreach (var project in projects)
{
if (ignoredProjects != null && ignoredProjects.Contains(project.ProjectUID))
{
Log.LogInformation($"{Method.Info()} Ignoring project {project.ProjectUID}; found in IgnoredProjects list.");
continue;
}
var projectRecord = _database.Find <MigrationProject>(Table.Projects, project.LegacyProjectID);
if (projectRecord == null)
{
Log.LogInformation($"{Method.Info()} Creating new migration record for project {project.ProjectUID}");
_database.Insert(new MigrationProject(project));
}
else
{
Log.LogInformation($"{Method.Info()} Found migration record for project {project.ProjectUID}");
// TODO Check completed=true & eligibleFiles > 0 && uploadedFiles=0; should retry.
if (projectRecord.MigrationState == MigrationState.Completed && !_reProcessSkippedFiles)
{
Log.LogInformation($"{Method.Info()} Skipping project {project.ProjectUID}, marked as COMPLETED");
continue;
}
if (projectRecord.MigrationState != MigrationState.Completed)
{
if (!_reProcessFailedProjects)
{
Log.LogInformation($"{Method.Info()} Not reprocessing {Enum.GetName(typeof(MigrationState), projectRecord.MigrationState)?.ToUpper()} project {project.ProjectUID}");
continue;
}
}
Log.LogInformation($"{Method.Info()} Resuming migration for project {project.ProjectUID}, marked as {Enum.GetName(typeof(MigrationState), projectRecord.MigrationState)?.ToUpper()}");
}
var job = new MigrationJob
{
Project = project,
IsRetryAttempt = projectRecord != null
};
if (projectsProcessed <= _capMigrationCount)
{
processedProjects.Add(job.Project.ProjectUID);
projectTasks.Add(MigrateProject(job));
}
if (projectTasks.Count <= THROTTLE_ASYNC_PROJECT_JOBS && projectsProcessed < _capMigrationCount - 1)
{
continue;
}
var completed = await Task.WhenAny(projectTasks);
projectTasks.Remove(completed);
_database.IncrementProjectMigrationCounter(project);
projectsProcessed += 1;
Log.LogInformation("Migration Progress:");
Log.LogInformation($" Processed: {projectsProcessed}");
Log.LogInformation($" In Flight: {projectTasks.Count}");
Log.LogInformation($" Remaining: {projectCount - projectsProcessed}");
if (projectsProcessed >= _capMigrationCount)
{
Log.LogInformation($"{Method.Info()} Reached maxium number of projects to process, exiting.");
break;
}
}
await Task.WhenAll(projectTasks);
// DIAGNOSTIC RUNTIME SWITCH
if (_saveFailedProjects)
{
// Create a recovery file of project uids for re processing
var failedProjectsLog = Path.Combine(_tempFolder, $"FailedProjects{DateTime.Now.Date.ToShortDateString().Replace('/', '-')}_{DateTime.Now.Hour}{DateTime.Now.Minute}{DateTime.Now.Second}.log");
var completedProjectsLog = Path.Combine(_tempFolder, $"Completed{DateTime.Now.Date.ToShortDateString().Replace('/', '-')}_{DateTime.Now.Hour}{DateTime.Now.Minute}{DateTime.Now.Second}.log");
if (!Directory.Exists(_tempFolder))
{
Directory.CreateDirectory(_tempFolder);
}
var allProjects = _database.GetTable <MigrationProject>(Table.Projects).ToList();
using (TextWriter streamWriterFailed = new StreamWriter(failedProjectsLog))
using (TextWriter streamWriterCompleted = new StreamWriter(completedProjectsLog))
{
foreach (var project in processedProjects)
{
var migrationProject = allProjects.FirstOrDefault(x => x.ProjectUid == project);
if (migrationProject == null)
{
continue;
}
if (migrationProject.MigrationState == MigrationState.Completed)
{
streamWriterCompleted.WriteLine($"{migrationProject.ProjectUid}");
continue;
}
var message = string.IsNullOrEmpty(migrationProject.MigrationStateMessage)
? null
: $" // {migrationProject.MigrationStateMessage}";
streamWriterFailed.WriteLine($"{migrationProject.ProjectUid}{message}");
}
}
}
// Set the final summary figures.
var completedCount = _database.Find <MigrationProject>(Table.Projects, x => x.MigrationState == MigrationState.Completed)
.Count();
_database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsSuccessful = completedCount);
var failedCount = _database.Find <MigrationProject>(Table.Projects, x => x.MigrationState == MigrationState.Failed)
.Count();
_database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsFailed = failedCount);
Log.LogInformation("Migration processing completed.");
}