private async Task ScanBucket(List <MigrationJob> adoptJobs, List <MigrationJob> configJobs, string bucketKeyOld) { _logger.LogInformation($"Scanning bucket {bucketKeyOld}"); MigrationBucketKeyProvider bucketProvider; ProjectService projectService; using (var scope = _serviceScopeFactory.CreateScope()) { bucketProvider = scope.ServiceProvider.GetService <MigrationBucketKeyProvider>(); projectService = scope.ServiceProvider.GetService <ProjectService>(); } OssBucket bucketOld = _bucketFactory.CreateBucket(bucketKeyOld); OssBucket bucketNew = _bucketFactory.CreateBucket(bucketProvider.GetBucketKeyFromOld(bucketKeyOld)); List <string> projectNamesNew = new List <string>(); try { List <string> projectNamesNewFromOss = (List <string>) await projectService.GetProjectNamesAsync(bucketNew); foreach (string projectName in projectNamesNewFromOss) { var ossAttributes = new OssAttributes(projectName); string metadataFile = ossAttributes.Metadata; // if metadata file is missing for project we consider that project not migrated if (await bucketNew.ObjectExistsAsync(metadataFile)) { projectNamesNew.Add(projectName); } } } catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound) { // swallow non existing item } // gather list of cache paramters files from the new bucket List <string> configKeysNew = new List <string>(); try { List <ObjectDetails> configODsNew = await bucketNew.GetObjectsAsync($"{ONC.CacheFolder}/"); foreach (ObjectDetails configODNew in configODsNew) { if (configODNew.ObjectKey.EndsWith(WebApplication.Utilities.LocalName.Parameters)) { configKeysNew.Add(configODNew.ObjectKey); } } } catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound) { // swallow non existing item } // gather projects to migrate List <string> projectNamesOld = (List <string>) await projectService.GetProjectNamesAsync(bucketOld); foreach (string projectName in projectNamesOld) { if (!projectNamesNew.Contains(projectName)) { // new project list does not contain old project => lets copy and adopt var ossAttributes = new OssAttributes(projectName); string metadataFile = ossAttributes.Metadata; try { ProjectMetadata projectMetadata = await bucketOld.DeserializeAsync <ProjectMetadata>(metadataFile); ProjectInfo projectInfo = new ProjectInfo(); projectInfo.Name = projectName; projectInfo.TopLevelAssembly = projectMetadata.TLA; MigrationJob migrationJob; using (var scope = _serviceScopeFactory.CreateScope()) { migrationJob = scope.ServiceProvider.GetService <MigrationJob>(); } migrationJob.SetJob(JobType.CopyAndAdopt, bucketOld, projectInfo, ONC.ProjectUrl(projectName)); adoptJobs.Add(migrationJob); } catch (Exception e) { _logger.LogError(e, $"Project {projectName} in bucket {bucketKeyOld} does not have metadata file. Skipping it."); } } // process cached configurations List <ObjectDetails> configODs = await bucketOld.GetObjectsAsync($"{ONC.CacheFolder}/{projectName}/"); foreach (ObjectDetails configOD in configODs) { string configKey = configOD.ObjectKey; if (configKey.EndsWith(WebApplication.Utilities.LocalName.Parameters)) { // calculate parameter hash based on new algorithmes InventorParameters parameters = await bucketOld.DeserializeAsync <InventorParameters>(configKey); string newHash = Crypto.GenerateParametersHashString(parameters); OSSObjectNameProvider onp = new OSSObjectNameProvider(projectName, newHash); configKey = onp.Parameters; if (!configKeysNew.Contains(configKey)) { ProjectInfo projectInfo = new ProjectInfo(); projectInfo.Name = projectName; MigrationJob migrationJob; using (var scope = _serviceScopeFactory.CreateScope()) { migrationJob = scope.ServiceProvider.GetService <MigrationJob>(); } migrationJob.SetJob(JobType.GenerateConfiguration, bucketOld, projectInfo, ONC.ProjectUrl(projectName), parameters); configJobs.Add(migrationJob); } } } } // check if any of migrated projects were deleted in old bucket // (user deleted them after migration started) foreach (string projectName in projectNamesNew) { if (!projectNamesOld.Contains(projectName)) { MigrationJob migrationJob; using (var scope = _serviceScopeFactory.CreateScope()) { migrationJob = scope.ServiceProvider.GetService <MigrationJob>(); } migrationJob.SetJob(JobType.RemoveNew, bucketNew, new ProjectInfo(projectName), null); adoptJobs.Add(migrationJob); } } }
private async Task ScanBucket(List <MigrationJob> migrationJobs, string bucketKeyOld) { OssBucket bucketOld = _bucketFactory.CreateBucket(bucketKeyOld); OssBucket bucketNew = _bucketFactory.CreateBucket(_bucketProvider.GetBucketKeyFromOld(bucketKeyOld)); List <string> projectNames = (List <string>) await _projectService.GetProjectNamesAsync(bucketOld); foreach (string projectName in projectNames) { var ossAttributes = new OssAttributes(projectName); string attributeFile = ossAttributes.Metadata; // check attributes file existance in new destination bucket if (await bucketNew.ObjectExistsAsync(attributeFile)) { continue; } ProjectMetadata projectMetadata = await bucketOld.DeserializeAsync <ProjectMetadata>(attributeFile); ProjectInfo projectInfo = new ProjectInfo(); projectInfo.Name = projectName; projectInfo.TopLevelAssembly = projectMetadata.TLA; migrationJobs.Add(new MigrationJob(JobType.CopyAndAdopt, bucketOld, projectInfo, ONC.ProjectUrl(projectName))); } }
private async Task ScanBucket(List <MigrationJob> migrationJobs, string bucketKeyOld) { OssBucket bucketOld = _bucketFactory.CreateBucket(bucketKeyOld); OssBucket bucketNew = _bucketFactory.CreateBucket(_bucketProvider.GetBucketKeyFromOld(bucketKeyOld)); List <string> projectNamesNew = new List <string>(); try { List <string> projectNamesNewFromOss = (List <string>) await _projectService.GetProjectNamesAsync(bucketNew); foreach (string projectName in projectNamesNewFromOss) { var ossAttributes = new OssAttributes(projectName); string metadataFile = ossAttributes.Metadata; // if metadata file is missing for project we consider that project not migrated if (await bucketNew.ObjectExistsAsync(metadataFile)) { projectNamesNew.Add(projectName); } } } catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound) { // swallow non existing item } List <string> projectNamesOld = (List <string>) await _projectService.GetProjectNamesAsync(bucketOld); foreach (string projectName in projectNamesOld) { if (!projectNamesNew.Contains(projectName)) { // new project list does not contain old project => lets copy and adopt var ossAttributes = new OssAttributes(projectName); string metadataFile = ossAttributes.Metadata; try { ProjectMetadata projectMetadata = await bucketOld.DeserializeAsync <ProjectMetadata>(metadataFile); ProjectInfo projectInfo = new ProjectInfo(); projectInfo.Name = projectName; projectInfo.TopLevelAssembly = projectMetadata.TLA; migrationJobs.Add(new MigrationJob(JobType.CopyAndAdopt, bucketOld, projectInfo, ONC.ProjectUrl(projectName))); } catch (Exception e) { _logger.LogError(e, $"Project {projectName} in bucket {bucketKeyOld} does not have metadata file. Skipping that."); } } } // check if any of migrated projects were deleted in old bucket // (user deleted them after migration started) foreach (string projectName in projectNamesNew) { if (!projectNamesOld.Contains(projectName)) { migrationJobs.Add(new MigrationJob(JobType.RemoveNew, bucketNew, new ProjectInfo(projectName), null)); } } }