/// <summary> /// Downloads the file from TCC and if successful uploads it through the Project service. /// </summary> private async Task <(bool success, FileDataSingleResult file)> MigrateFile(ImportedFileDescriptor file, Project project) { if (_ignoredFiles != null && _ignoredFiles.Contains(file.ImportedFileUid)) { Log.LogWarning($"{Method.Info()} Migrating file '{file.Name}', Uid: {file.ImportedFileUid} aborted, found in exclusion list."); return(success : false, file : null); } Log.LogInformation($"{Method.In()} Migrating file '{file.Name}', Uid: {file.ImportedFileUid}"); string tempFileName; using (var fileContents = await FileRepo.GetFile(_fileSpaceId, $"{file.Path}/{file.Name}")) { _database.Update(file.LegacyFileId, (MigrationFile x) => x.MigrationState = MigrationState.InProgress, Table.Files); if (fileContents == null) { var message = $"Failed to fetch file '{file.Name}' ({file.LegacyFileId}), not found"; _database.Update(file.LegacyFileId, (MigrationFile x) => x.MigrationState = MigrationState.FileNotFound, Table.Files); _database.Insert(new MigrationMessage(file.ProjectUid, message)); Log.LogWarning($"{Method.Out()} {message}"); return(success : true, file : null); } var tempPath = Path.Combine(_tempFolder, file.CustomerUid, file.ProjectUid, file.ImportedFileUid); Directory.CreateDirectory(tempPath); tempFileName = Path.Combine(tempPath, file.Name); Log.LogInformation($"{Method.Info()} Creating temporary file '{tempFileName}' for file {file.ImportedFileUid}"); if (_downloadProjectFiles) { using (var tempFile = new FileStream(tempFileName, FileMode.Create)) { fileContents.CopyTo(tempFile); _database.Update( file.LegacyFileId, (MigrationFile x) => { // ReSharper disable once AccessToDisposedClosure x.Length = tempFile.Length; }, tableName: Table.Files); } } } var result = new FileDataSingleResult(); if (_downloadProjectFiles && _uploadProjectFiles) { Log.LogInformation($"{Method.Info()} Uploading file {file.ImportedFileUid}"); result = ImportFile.SendRequestToFileImportV4( _uploadFileApiUrl, file, tempFileName, new ImportOptions(HttpMethod.Put)); _database.Update(file.LegacyFileId, (MigrationFile x) => x.MigrationState = MigrationState.Completed, Table.Files); _database.Update(project.LegacyProjectID, (MigrationProject x) => x.UploadedFileCount += 1, Table.Projects); } else { var skippedMessage = $"Skipped because DOWNLOAD_PROJECT_FILES={_downloadProjectFiles} && UPLOAD_PROJECT_FILES={_uploadProjectFiles}"; _database.Update(file.LegacyFileId, (MigrationFile x) => { x.MigrationState = MigrationState.Skipped; x.MigrationStateMessage = skippedMessage; }, Table.Files); Log.LogDebug($"{Method.Info("DEBUG")} {skippedMessage}"); } Log.LogInformation($"{Method.Out()} File {file.ImportedFileUid} update result {result.Code} {result.Message}"); return(success : true, file : result); }
/// <summary> /// Migrate all elgible files for a given project. /// </summary> private async Task <bool> MigrateProject(MigrationJob job) { var migrationResult = MigrationState.Unknown; var migrationStateMessage = ""; try { Log.LogInformation($"{Method.In()} Migrating project {job.Project.ProjectUID}, Name: '{job.Project.Name}'"); _database.SetMigrationState(job, MigrationState.InProgress, null); // Resolve coordinate system file first; all projects regardless of whether they have files need to have // their calibration file present in DataOcean post migration. var result = await DcFileAgent.ResolveProjectCoordinateSystemFile(job); if (!result) { migrationResult = MigrationState.Failed; migrationStateMessage = "Unable to resolve coordinate system file"; Log.LogError($"{Method.Info()} {migrationStateMessage} for project {job.Project.ProjectUID}, aborting project migration"); return(false); } // Resolve imported files for current project. var filesResult = await ImportFile.GetImportedFilesFromWebApi($"{_importedFileApiUrl}?projectUid={job.Project.ProjectUID}", job.Project); if (filesResult == null) { Log.LogInformation($"{Method.Info()} Failed to fetch imported files for project {job.Project.ProjectUID}, aborting project migration"); _database.SetMigrationState(job, MigrationState.Failed, "Failed to fetch imported file list"); migrationStateMessage = "Failed to fetch imported file list"; return(false); } if (filesResult.ImportedFileDescriptors == null || filesResult.ImportedFileDescriptors.Count == 0) { Log.LogInformation($"{Method.Info()} Project {job.Project.ProjectUID} contains no imported files, aborting project migration"); _database.SetMigrationState(job, MigrationState.Skipped, "No imported files"); _database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsWithNoFiles += 1); migrationStateMessage = "Project contains no imported files"; migrationResult = MigrationState.Completed; } else { // We have files, and a valid coordinate system, continue processing. var selectedFiles = filesResult.ImportedFileDescriptors .Where(f => MigrationFileTypes.Contains(f.ImportedFileType)) .ToList(); _database.Update(job.Project.LegacyProjectID, (MigrationProject x) => { x.TotalFileCount = filesResult.ImportedFileDescriptors.Count; x.EligibleFileCount = selectedFiles.Count; }, Table.Projects); Log.LogInformation($"{Method.Info()} Found {selectedFiles.Count} eligible files out of {filesResult.ImportedFileDescriptors.Count} total to migrate for {job.Project.ProjectUID}"); if (selectedFiles.Count == 0) { Log.LogInformation($"{Method.Info()} Project {job.Project.ProjectUID} contains no eligible files, skipping project migration"); _database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsWithNoEligibleFiles += 1); migrationStateMessage = "Project contains no eligible files"; migrationResult = MigrationState.Completed; } var fileTasks = new List <Task <(bool, FileDataSingleResult)> >(); foreach (var file in selectedFiles) { // Check to sort out bad data; found in development database. if (file.CustomerUid != job.Project.CustomerUID) { Log.LogError($"{Method.Info("ERROR")} CustomerUid ({file.CustomerUid}) for ImportedFile ({file.ImportedFileUid}) doesn't match associated project: {job.Project.ProjectUID}"); continue; } var migrationFile = _database.Find <MigrationFile>(Table.Files, file.LegacyFileId); if (migrationFile == null) { _database.Insert(new MigrationFile(file), Table.Files); } else { if (migrationFile.MigrationState == MigrationState.Completed || migrationFile.MigrationState == MigrationState.Skipped && !_reProcessSkippedFiles) { Log.LogInformation($"{Method.Info()} Skipping file {file.ImportedFileUid}, migrationState={Enum.GetName(typeof(MigrationState), migrationFile.MigrationState)?.ToUpper()} and REPROCESS_SKIPPED_FILES={_reProcessSkippedFiles}"); continue; } } var migrationResultObj = MigrateFile(file, job.Project); fileTasks.Add(migrationResultObj); if (fileTasks.Count != THROTTLE_ASYNC_FILE_UPLOAD_JOBS) { continue; } var completed = await Task.WhenAny(fileTasks); fileTasks.Remove(completed); } await Task.WhenAll(fileTasks); var importedFilesResult = fileTasks.All(t => t.Result.Item1); migrationResult = importedFilesResult ? MigrationState.Completed : MigrationState.Failed; if (!_uploadProjectFiles) { migrationResult = MigrationState.Unknown; } Log.LogInformation($"{Method.Out()} Project '{job.Project.Name}' ({job.Project.ProjectUID}) {(importedFilesResult ? "succeeded" : "failed")}"); migrationStateMessage = importedFilesResult ? "Success" : "failed"; return(importedFilesResult); } } catch (Exception exception) { Log.LogError(exception, $"{Method.Info()} Error processing project {job.Project.ProjectUID}"); } finally { _database.SetMigrationState(job, migrationResult, migrationStateMessage); Action <MigrationInfo> migrationResultAction; switch (migrationResult) { case MigrationState.Skipped: migrationResultAction = x => x.ProjectsSkipped += 1; break; case MigrationState.Completed: migrationResultAction = x => x.ProjectsCompleted += 1; break; case MigrationState.Failed: migrationResultAction = x => x.ProjectsFailed += 1; break; default: if (!_uploadProjectFiles) { migrationResultAction = x => x.ProjectsSkipped += 1; break; } throw new Exception($"Invalid migrationResult state for project {job.Project.ProjectUID}"); } _database.Update(MigrationInfoId, migrationResultAction); } return(false); }