/// <summary>
/// Downloads the file from TCC and if successful uploads it through the Project service.
/// </summary>
private async Task <(bool success, FileDataSingleResult file)> MigrateFile(ImportedFileDescriptor file, Project project)
{
if (_ignoredFiles != null && _ignoredFiles.Contains(file.ImportedFileUid))
{
Log.LogWarning($"{Method.Info()} Migrating file '{file.Name}', Uid: {file.ImportedFileUid} aborted, found in exclusion list.");
return(success : false, file : null);
}
Log.LogInformation($"{Method.In()} Migrating file '{file.Name}', Uid: {file.ImportedFileUid}");
string tempFileName;
using (var fileContents = await FileRepo.GetFile(_fileSpaceId, $"{file.Path}/{file.Name}"))
{
_database.Update(file.LegacyFileId, (MigrationFile x) => x.MigrationState = MigrationState.InProgress, Table.Files);
if (fileContents == null)
{
var message = $"Failed to fetch file '{file.Name}' ({file.LegacyFileId}), not found";
_database.Update(file.LegacyFileId, (MigrationFile x) => x.MigrationState = MigrationState.FileNotFound, Table.Files);
_database.Insert(new MigrationMessage(file.ProjectUid, message));
Log.LogWarning($"{Method.Out()} {message}");
return(success : true, file : null);
}
var tempPath = Path.Combine(_tempFolder, file.CustomerUid, file.ProjectUid, file.ImportedFileUid);
Directory.CreateDirectory(tempPath);
tempFileName = Path.Combine(tempPath, file.Name);
Log.LogInformation($"{Method.Info()} Creating temporary file '{tempFileName}' for file {file.ImportedFileUid}");
if (_downloadProjectFiles)
{
using (var tempFile = new FileStream(tempFileName, FileMode.Create))
{
fileContents.CopyTo(tempFile);
_database.Update(
file.LegacyFileId, (MigrationFile x) =>
{
// ReSharper disable once AccessToDisposedClosure
x.Length = tempFile.Length;
},
tableName: Table.Files);
}
}
}
var result = new FileDataSingleResult();
if (_downloadProjectFiles && _uploadProjectFiles)
{
Log.LogInformation($"{Method.Info()} Uploading file {file.ImportedFileUid}");
result = ImportFile.SendRequestToFileImportV4(
_uploadFileApiUrl,
file,
tempFileName,
new ImportOptions(HttpMethod.Put));
_database.Update(file.LegacyFileId, (MigrationFile x) => x.MigrationState = MigrationState.Completed, Table.Files);
_database.Update(project.LegacyProjectID, (MigrationProject x) => x.UploadedFileCount += 1, Table.Projects);
}
else
{
var skippedMessage = $"Skipped because DOWNLOAD_PROJECT_FILES={_downloadProjectFiles} && UPLOAD_PROJECT_FILES={_uploadProjectFiles}";
_database.Update(file.LegacyFileId, (MigrationFile x) =>
{
x.MigrationState = MigrationState.Skipped;
x.MigrationStateMessage = skippedMessage;
}, Table.Files);
Log.LogDebug($"{Method.Info("DEBUG")} {skippedMessage}");
}
Log.LogInformation($"{Method.Out()} File {file.ImportedFileUid} update result {result.Code} {result.Message}");
return(success : true, file : result);
}
/// <summary>
/// Migrate all elgible files for a given project.
/// </summary>
private async Task <bool> MigrateProject(MigrationJob job)
{
var migrationResult = MigrationState.Unknown;
var migrationStateMessage = "";
try
{
Log.LogInformation($"{Method.In()} Migrating project {job.Project.ProjectUID}, Name: '{job.Project.Name}'");
_database.SetMigrationState(job, MigrationState.InProgress, null);
// Resolve coordinate system file first; all projects regardless of whether they have files need to have
// their calibration file present in DataOcean post migration.
var result = await DcFileAgent.ResolveProjectCoordinateSystemFile(job);
if (!result)
{
migrationResult = MigrationState.Failed;
migrationStateMessage = "Unable to resolve coordinate system file";
Log.LogError($"{Method.Info()} {migrationStateMessage} for project {job.Project.ProjectUID}, aborting project migration");
return(false);
}
// Resolve imported files for current project.
var filesResult = await ImportFile.GetImportedFilesFromWebApi($"{_importedFileApiUrl}?projectUid={job.Project.ProjectUID}", job.Project);
if (filesResult == null)
{
Log.LogInformation($"{Method.Info()} Failed to fetch imported files for project {job.Project.ProjectUID}, aborting project migration");
_database.SetMigrationState(job, MigrationState.Failed, "Failed to fetch imported file list");
migrationStateMessage = "Failed to fetch imported file list";
return(false);
}
if (filesResult.ImportedFileDescriptors == null || filesResult.ImportedFileDescriptors.Count == 0)
{
Log.LogInformation($"{Method.Info()} Project {job.Project.ProjectUID} contains no imported files, aborting project migration");
_database.SetMigrationState(job, MigrationState.Skipped, "No imported files");
_database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsWithNoFiles += 1);
migrationStateMessage = "Project contains no imported files";
migrationResult = MigrationState.Completed;
}
else
{
// We have files, and a valid coordinate system, continue processing.
var selectedFiles = filesResult.ImportedFileDescriptors
.Where(f => MigrationFileTypes.Contains(f.ImportedFileType))
.ToList();
_database.Update(job.Project.LegacyProjectID, (MigrationProject x) =>
{
x.TotalFileCount = filesResult.ImportedFileDescriptors.Count;
x.EligibleFileCount = selectedFiles.Count;
}, Table.Projects);
Log.LogInformation($"{Method.Info()} Found {selectedFiles.Count} eligible files out of {filesResult.ImportedFileDescriptors.Count} total to migrate for {job.Project.ProjectUID}");
if (selectedFiles.Count == 0)
{
Log.LogInformation($"{Method.Info()} Project {job.Project.ProjectUID} contains no eligible files, skipping project migration");
_database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsWithNoEligibleFiles += 1);
migrationStateMessage = "Project contains no eligible files";
migrationResult = MigrationState.Completed;
}
var fileTasks = new List <Task <(bool, FileDataSingleResult)> >();
foreach (var file in selectedFiles)
{
// Check to sort out bad data; found in development database.
if (file.CustomerUid != job.Project.CustomerUID)
{
Log.LogError($"{Method.Info("ERROR")} CustomerUid ({file.CustomerUid}) for ImportedFile ({file.ImportedFileUid}) doesn't match associated project: {job.Project.ProjectUID}");
continue;
}
var migrationFile = _database.Find <MigrationFile>(Table.Files, file.LegacyFileId);
if (migrationFile == null)
{
_database.Insert(new MigrationFile(file), Table.Files);
}
else
{
if (migrationFile.MigrationState == MigrationState.Completed ||
migrationFile.MigrationState == MigrationState.Skipped && !_reProcessSkippedFiles)
{
Log.LogInformation($"{Method.Info()} Skipping file {file.ImportedFileUid}, migrationState={Enum.GetName(typeof(MigrationState), migrationFile.MigrationState)?.ToUpper()} and REPROCESS_SKIPPED_FILES={_reProcessSkippedFiles}");
continue;
}
}
var migrationResultObj = MigrateFile(file, job.Project);
fileTasks.Add(migrationResultObj);
if (fileTasks.Count != THROTTLE_ASYNC_FILE_UPLOAD_JOBS)
{
continue;
}
var completed = await Task.WhenAny(fileTasks);
fileTasks.Remove(completed);
}
await Task.WhenAll(fileTasks);
var importedFilesResult = fileTasks.All(t => t.Result.Item1);
migrationResult = importedFilesResult ? MigrationState.Completed : MigrationState.Failed;
if (!_uploadProjectFiles)
{
migrationResult = MigrationState.Unknown;
}
Log.LogInformation($"{Method.Out()} Project '{job.Project.Name}' ({job.Project.ProjectUID}) {(importedFilesResult ? "succeeded" : "failed")}");
migrationStateMessage = importedFilesResult ? "Success" : "failed";
return(importedFilesResult);
}
}
catch (Exception exception)
{
Log.LogError(exception, $"{Method.Info()} Error processing project {job.Project.ProjectUID}");
}
finally
{
_database.SetMigrationState(job, migrationResult, migrationStateMessage);
Action <MigrationInfo> migrationResultAction;
switch (migrationResult)
{
case MigrationState.Skipped: migrationResultAction = x => x.ProjectsSkipped += 1; break;
case MigrationState.Completed: migrationResultAction = x => x.ProjectsCompleted += 1; break;
case MigrationState.Failed: migrationResultAction = x => x.ProjectsFailed += 1; break;
default:
if (!_uploadProjectFiles)
{
migrationResultAction = x => x.ProjectsSkipped += 1;
break;
}
throw new Exception($"Invalid migrationResult state for project {job.Project.ProjectUID}");
}
_database.Update(MigrationInfoId, migrationResultAction);
}
return(false);
}