Beispiel #1
0
        /// <summary>
        /// Downloads the file from TCC and if successful uploads it through the Project service.
        /// </summary>
        private async Task <(bool success, FileDataSingleResult file)> MigrateFile(ImportedFileDescriptor file, Project project)
        {
            if (_ignoredFiles != null && _ignoredFiles.Contains(file.ImportedFileUid))
            {
                Log.LogWarning($"{Method.Info()} Migrating file '{file.Name}', Uid: {file.ImportedFileUid} aborted, found in exclusion list.");

                return(success : false, file : null);
            }

            Log.LogInformation($"{Method.In()} Migrating file '{file.Name}', Uid: {file.ImportedFileUid}");

            string tempFileName;

            using (var fileContents = await FileRepo.GetFile(_fileSpaceId, $"{file.Path}/{file.Name}"))
            {
                _database.Update(file.LegacyFileId, (MigrationFile x) => x.MigrationState = MigrationState.InProgress, Table.Files);

                if (fileContents == null)
                {
                    var message = $"Failed to fetch file '{file.Name}' ({file.LegacyFileId}), not found";
                    _database.Update(file.LegacyFileId, (MigrationFile x) => x.MigrationState = MigrationState.FileNotFound, Table.Files);
                    _database.Insert(new MigrationMessage(file.ProjectUid, message));

                    Log.LogWarning($"{Method.Out()} {message}");

                    return(success : true, file : null);
                }

                var tempPath = Path.Combine(_tempFolder, file.CustomerUid, file.ProjectUid, file.ImportedFileUid);
                Directory.CreateDirectory(tempPath);

                tempFileName = Path.Combine(tempPath, file.Name);

                Log.LogInformation($"{Method.Info()} Creating temporary file '{tempFileName}' for file {file.ImportedFileUid}");

                if (_downloadProjectFiles)
                {
                    using (var tempFile = new FileStream(tempFileName, FileMode.Create))
                    {
                        fileContents.CopyTo(tempFile);

                        _database.Update(
                            file.LegacyFileId, (MigrationFile x) =>
                        {
                            // ReSharper disable once AccessToDisposedClosure
                            x.Length = tempFile.Length;
                        },
                            tableName: Table.Files);
                    }
                }
            }

            var result = new FileDataSingleResult();

            if (_downloadProjectFiles && _uploadProjectFiles)
            {
                Log.LogInformation($"{Method.Info()} Uploading file {file.ImportedFileUid}");

                result = ImportFile.SendRequestToFileImportV4(
                    _uploadFileApiUrl,
                    file,
                    tempFileName,
                    new ImportOptions(HttpMethod.Put));

                _database.Update(file.LegacyFileId, (MigrationFile x) => x.MigrationState              = MigrationState.Completed, Table.Files);
                _database.Update(project.LegacyProjectID, (MigrationProject x) => x.UploadedFileCount += 1, Table.Projects);
            }
            else
            {
                var skippedMessage = $"Skipped because DOWNLOAD_PROJECT_FILES={_downloadProjectFiles} && UPLOAD_PROJECT_FILES={_uploadProjectFiles}";

                _database.Update(file.LegacyFileId, (MigrationFile x) =>
                {
                    x.MigrationState        = MigrationState.Skipped;
                    x.MigrationStateMessage = skippedMessage;
                }, Table.Files);

                Log.LogDebug($"{Method.Info("DEBUG")} {skippedMessage}");
            }

            Log.LogInformation($"{Method.Out()} File {file.ImportedFileUid} update result {result.Code} {result.Message}");

            return(success : true, file : result);
        }
Beispiel #2
0
        /// <summary>
        /// Migrate all elgible files for a given project.
        /// </summary>
        private async Task <bool> MigrateProject(MigrationJob job)
        {
            var migrationResult       = MigrationState.Unknown;
            var migrationStateMessage = "";

            try
            {
                Log.LogInformation($"{Method.In()} Migrating project {job.Project.ProjectUID}, Name: '{job.Project.Name}'");
                _database.SetMigrationState(job, MigrationState.InProgress, null);

                // Resolve coordinate system file first; all projects regardless of whether they have files need to have
                // their calibration file present in DataOcean post migration.
                var result = await DcFileAgent.ResolveProjectCoordinateSystemFile(job);

                if (!result)
                {
                    migrationResult       = MigrationState.Failed;
                    migrationStateMessage = "Unable to resolve coordinate system file";

                    Log.LogError($"{Method.Info()} {migrationStateMessage} for project {job.Project.ProjectUID}, aborting project migration");

                    return(false);
                }

                // Resolve imported files for current project.
                var filesResult = await ImportFile.GetImportedFilesFromWebApi($"{_importedFileApiUrl}?projectUid={job.Project.ProjectUID}", job.Project);

                if (filesResult == null)
                {
                    Log.LogInformation($"{Method.Info()} Failed to fetch imported files for project {job.Project.ProjectUID}, aborting project migration");
                    _database.SetMigrationState(job, MigrationState.Failed, "Failed to fetch imported file list");

                    migrationStateMessage = "Failed to fetch imported file list";
                    return(false);
                }

                if (filesResult.ImportedFileDescriptors == null || filesResult.ImportedFileDescriptors.Count == 0)
                {
                    Log.LogInformation($"{Method.Info()} Project {job.Project.ProjectUID} contains no imported files, aborting project migration");

                    _database.SetMigrationState(job, MigrationState.Skipped, "No imported files");
                    _database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsWithNoFiles += 1);

                    migrationStateMessage = "Project contains no imported files";
                    migrationResult       = MigrationState.Completed;
                }
                else
                {
                    // We have files, and a valid coordinate system, continue processing.
                    var selectedFiles = filesResult.ImportedFileDescriptors
                                        .Where(f => MigrationFileTypes.Contains(f.ImportedFileType))
                                        .ToList();

                    _database.Update(job.Project.LegacyProjectID, (MigrationProject x) =>
                    {
                        x.TotalFileCount    = filesResult.ImportedFileDescriptors.Count;
                        x.EligibleFileCount = selectedFiles.Count;
                    }, Table.Projects);

                    Log.LogInformation($"{Method.Info()} Found {selectedFiles.Count} eligible files out of {filesResult.ImportedFileDescriptors.Count} total to migrate for {job.Project.ProjectUID}");

                    if (selectedFiles.Count == 0)
                    {
                        Log.LogInformation($"{Method.Info()} Project {job.Project.ProjectUID} contains no eligible files, skipping project migration");

                        _database.Update(MigrationInfoId, (MigrationInfo x) => x.ProjectsWithNoEligibleFiles += 1);

                        migrationStateMessage = "Project contains no eligible files";
                        migrationResult       = MigrationState.Completed;
                    }

                    var fileTasks = new List <Task <(bool, FileDataSingleResult)> >();

                    foreach (var file in selectedFiles)
                    {
                        // Check to sort out bad data; found in development database.
                        if (file.CustomerUid != job.Project.CustomerUID)
                        {
                            Log.LogError($"{Method.Info("ERROR")} CustomerUid ({file.CustomerUid}) for ImportedFile ({file.ImportedFileUid}) doesn't match associated project: {job.Project.ProjectUID}");
                            continue;
                        }

                        var migrationFile = _database.Find <MigrationFile>(Table.Files, file.LegacyFileId);

                        if (migrationFile == null)
                        {
                            _database.Insert(new MigrationFile(file), Table.Files);
                        }
                        else
                        {
                            if (migrationFile.MigrationState == MigrationState.Completed ||
                                migrationFile.MigrationState == MigrationState.Skipped && !_reProcessSkippedFiles)
                            {
                                Log.LogInformation($"{Method.Info()} Skipping file {file.ImportedFileUid}, migrationState={Enum.GetName(typeof(MigrationState), migrationFile.MigrationState)?.ToUpper()} and REPROCESS_SKIPPED_FILES={_reProcessSkippedFiles}");

                                continue;
                            }
                        }

                        var migrationResultObj = MigrateFile(file, job.Project);

                        fileTasks.Add(migrationResultObj);

                        if (fileTasks.Count != THROTTLE_ASYNC_FILE_UPLOAD_JOBS)
                        {
                            continue;
                        }

                        var completed = await Task.WhenAny(fileTasks);

                        fileTasks.Remove(completed);
                    }

                    await Task.WhenAll(fileTasks);

                    var importedFilesResult = fileTasks.All(t => t.Result.Item1);

                    migrationResult = importedFilesResult ? MigrationState.Completed : MigrationState.Failed;
                    if (!_uploadProjectFiles)
                    {
                        migrationResult = MigrationState.Unknown;
                    }

                    Log.LogInformation($"{Method.Out()} Project '{job.Project.Name}' ({job.Project.ProjectUID}) {(importedFilesResult ? "succeeded" : "failed")}");

                    migrationStateMessage = importedFilesResult ? "Success" : "failed";
                    return(importedFilesResult);
                }
            }
            catch (Exception exception)
            {
                Log.LogError(exception, $"{Method.Info()} Error processing project {job.Project.ProjectUID}");
            }
            finally
            {
                _database.SetMigrationState(job, migrationResult, migrationStateMessage);

                Action <MigrationInfo> migrationResultAction;

                switch (migrationResult)
                {
                case MigrationState.Skipped: migrationResultAction = x => x.ProjectsSkipped += 1; break;

                case MigrationState.Completed: migrationResultAction = x => x.ProjectsCompleted += 1; break;

                case MigrationState.Failed: migrationResultAction = x => x.ProjectsFailed += 1; break;

                default:
                    if (!_uploadProjectFiles)
                    {
                        migrationResultAction = x => x.ProjectsSkipped += 1;
                        break;
                    }

                    throw new Exception($"Invalid migrationResult state for project {job.Project.ProjectUID}");
                }

                _database.Update(MigrationInfoId, migrationResultAction);
            }

            return(false);
        }