public MigrationJob(JobType jobTypeParam, OssBucket bucketParam, ProjectInfo projectInfoParam, string projectUrlParam)
{
jobType = jobTypeParam;
bucket = bucketParam;
projectInfo = projectInfoParam;
projectUrl = projectUrlParam;
}
private async Task ScanBucket(List <MigrationJob> migrationJobs, string bucketKeyOld)
{
OssBucket bucketOld = _bucketFactory.CreateBucket(bucketKeyOld);
OssBucket bucketNew = _bucketFactory.CreateBucket(_bucketProvider.GetBucketKeyFromOld(bucketKeyOld));
List <string> projectNames = (List <string>) await _projectService.GetProjectNamesAsync(bucketOld);
foreach (string projectName in projectNames)
{
var ossAttributes = new OssAttributes(projectName);
string attributeFile = ossAttributes.Metadata;
// check attributes file existance in new destination bucket
if (await bucketNew.ObjectExistsAsync(attributeFile))
{
continue;
}
ProjectMetadata projectMetadata = await bucketOld.DeserializeAsync <ProjectMetadata>(attributeFile);
ProjectInfo projectInfo = new ProjectInfo();
projectInfo.Name = projectName;
projectInfo.TopLevelAssembly = projectMetadata.TLA;
migrationJobs.Add(new MigrationJob(JobType.CopyAndAdopt, bucketOld, projectInfo, ONC.ProjectUrl(projectName)));
}
}
private async Task CopyAndAdopt(string logId)
{
_bucketProvider.SetBucketKeyFromOld(bucket.BucketKey);
_logger.LogInformation($"{logId}: Processing new project {projectInfo.Name} in bucket {bucket.BucketKey}");
OssBucket bucketNew = await _userResolver.GetBucketAsync(true);
string signedUrlOld = await bucket.CreateSignedUrlAsync(projectUrl, ObjectAccess.Read);
string signedUrlNew = await bucketNew.CreateSignedUrlAsync(projectUrl, ObjectAccess.ReadWrite);
try
{
await _projectWork.FileTransferAsync(signedUrlOld, signedUrlNew);
}
catch (Exception e)
{
_logger.LogError(e, $"{logId}: Project {projectInfo.Name} cannot be copied.");
return;
}
try
{
await _projectWork.AdoptAsync(projectInfo, signedUrlNew);
_logger.LogInformation($"{logId}: Project {projectInfo.Name} was adopted");
}
catch (Exception e)
{
_logger.LogError(e, $"{logId}: Project {projectInfo.Name} was not adopted");
}
}
public async Task Migrate(List <MigrationJob> migrationJobs)
{
foreach (MigrationJob job in migrationJobs)
{
_bucketProvider.SetBucketKeyFromOld(job.bucketOld.BucketKey);
OssBucket bucketNew = await _userResolver.GetBucketAsync(true);
string signedUrlOld = await job.bucketOld.CreateSignedUrlAsync(job.projectUrl, ObjectAccess.Read);
string signedUrlNew = await bucketNew.CreateSignedUrlAsync(job.projectUrl, ObjectAccess.ReadWrite);
try
{
await _projectWork.FileTransferAsync(signedUrlOld, signedUrlNew);
}
catch (Exception e)
{
_logger.LogError("Project " + job.projectInfo.Name + " cannot be copied\nException:" + e.Message);
continue;
}
try
{
await _projectWork.AdoptAsync(job.projectInfo, signedUrlNew);
_logger.LogInformation("Project " + job.projectInfo.Name + " was adopted");
}
catch (Exception e)
{
_logger.LogError("Project " + job.projectInfo.Name + " was not adopted\nException:" + e.Message);
}
}
}
public void SetJob(JobType jobTypeParam, OssBucket bucketParam, ProjectInfo projectInfoParam, string projectUrlParam, InventorParameters parametersParam = null)
{
jobType = jobTypeParam;
bucket = bucketParam;
projectInfo = projectInfoParam;
projectUrl = projectUrlParam;
parameters = parametersParam;
}
private async Task ScanBucket(List <MigrationJob> migrationJobs, string bucketKeyOld)
{
OssBucket bucketOld = _bucketFactory.CreateBucket(bucketKeyOld);
OssBucket bucketNew = _bucketFactory.CreateBucket(_bucketProvider.GetBucketKeyFromOld(bucketKeyOld));
List <string> projectNamesNew = new List <string>();
try
{
List <string> projectNamesNewFromOss = (List <string>) await _projectService.GetProjectNamesAsync(bucketNew);
foreach (string projectName in projectNamesNewFromOss)
{
var ossAttributes = new OssAttributes(projectName);
string metadataFile = ossAttributes.Metadata;
// if metadata file is missing for project we consider that project not migrated
if (await bucketNew.ObjectExistsAsync(metadataFile))
{
projectNamesNew.Add(projectName);
}
}
}
catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound)
{
// swallow non existing item
}
List <string> projectNamesOld = (List <string>) await _projectService.GetProjectNamesAsync(bucketOld);
foreach (string projectName in projectNamesOld)
{
if (!projectNamesNew.Contains(projectName))
{
// new project list does not contain old project => lets copy and adopt
var ossAttributes = new OssAttributes(projectName);
string metadataFile = ossAttributes.Metadata;
ProjectMetadata projectMetadata = await bucketOld.DeserializeAsync <ProjectMetadata>(metadataFile);
ProjectInfo projectInfo = new ProjectInfo();
projectInfo.Name = projectName;
projectInfo.TopLevelAssembly = projectMetadata.TLA;
migrationJobs.Add(new MigrationJob(JobType.CopyAndAdopt, bucketOld, projectInfo, ONC.ProjectUrl(projectName)));
}
}
// check if any of migrated projects were deleted in old bucket
// (user deleted them after migration started)
foreach (string projectName in projectNamesNew)
{
if (!projectNamesOld.Contains(projectName))
{
migrationJobs.Add(new MigrationJob(JobType.RemoveNew, bucketNew, new ProjectInfo(projectName), null));
}
}
}
public DataMigrationController(ILogger <DataMigrationController> logger, IOptions <DefaultProjectsConfiguration> optionsAccessor,
ProjectWork projectWork, UserResolver userResolver, IForgeOSS forgeOSS, LocalCache localCache)
{
_logger = logger;
_userResolver = userResolver;
_bucket = _userResolver.AnonymousBucket;
_projectWork = projectWork;
_defaultProjectsConfiguration = optionsAccessor.Value;
_forgeOSS = forgeOSS;
_localCache = localCache;
}
/// <summary>
/// Get list of project names for a bucket.
/// </summary>
public async Task <ICollection <string> > GetProjectNamesAsync(OssBucket bucket = null)
{
bucket ??= await _userResolver.GetBucketAsync(true);
var objectDetails = (await bucket.GetObjectsAsync(ONC.ProjectsMask));
var projectNames = objectDetails
.Select(objDetails => ONC.ToProjectName(objDetails.ObjectKey))
.ToList();
return(projectNames);
}
/// <summary>
/// Constructor.
/// </summary>
public Initializer(ILogger <Initializer> logger, FdaClient fdaClient, IOptions <DefaultProjectsConfiguration> optionsAccessor,
ProjectWork projectWork, UserResolver userResolver, LocalCache localCache)
{
_logger = logger;
_fdaClient = fdaClient;
_projectWork = projectWork;
_userResolver = userResolver;
_localCache = localCache;
_defaultProjectsConfiguration = optionsAccessor.Value;
// bucket for anonymous user
_bucket = _userResolver.AnonymousBucket;
}
public async Task DeleteProjects(ICollection <string> projectNameList, OssBucket bucket = null)
{
bucket ??= await _userResolver.GetBucketAsync(true);
_logger.LogInformation($"deleting projects [{string.Join(", ", projectNameList)}] from bucket {bucket.BucketKey}");
// collect all oss objects for all provided projects
var tasks = new List <Task>();
foreach (var projectName in projectNameList)
{
tasks.Add(bucket.DeleteObjectAsync(Project.ExactOssName(projectName)));
foreach (var searchMask in ONC.ProjectFileMasks(projectName))
{
var objects = await bucket.GetObjectsAsync(searchMask);
foreach (var objectDetail in objects)
{
tasks.Add(bucket.DeleteObjectAsync(objectDetail.ObjectKey));
}
}
}
// delete the OSS objects
await Task.WhenAll(tasks);
for (var i = 0; i < tasks.Count; i++)
{
if (tasks[i].IsFaulted)
{
_logger.LogError($"Failed to delete file #{i}");
}
}
// delete local cache for all provided projects
foreach (var projectName in projectNameList)
{
var projectStorage = await _userResolver.GetProjectStorageAsync(projectName, ensureDir : false);
projectStorage.DeleteLocal();
}
}
private async Task GenerateConfiguration(string logId)
{
_bucketProvider.SetBucketKeyFromOld(bucket.BucketKey);
_logger.LogInformation($"{logId}: Generating config for project {projectInfo.Name} in bucket {bucket.BucketKey}");
OssBucket bucketNew = await _userResolver.GetBucketAsync();
try
{
await _projectWork.DoSmartUpdateAsync(parameters, projectInfo.Name);
_logger.LogInformation($"{logId}: Configuration {parameters} for project {projectInfo.Name} was generated.");
}
catch (Exception e)
{
_logger.LogError(e, $"{logId}: Configuration {parameters} for project {projectInfo.Name} was NOT generated.");
}
return;
}
public async Task <string> TransferProjectToOssAsync(OssBucket bucket, DefaultProjectConfiguration projectConfig)
{
_logger.LogInformation($"Bucket {bucket.BucketKey} created");
var projectUrl = projectConfig.Url;
var project = await _userResolver.GetProjectAsync(projectConfig.Name);
_logger.LogInformation($"Launching 'TransferData' for {projectUrl}");
// OSS bucket might be not ready yet, so repeat attempts
string signedUrl = await _waitForBucketPolicy.ExecuteAsync(async() =>
await bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite));
// TransferData from outside URL to temporary oss url
await _projectWork.FileTransferAsync(projectUrl, signedUrl);
_logger.LogInformation($"'TransferData' for {projectUrl} is done.");
return(signedUrl);
}
/// <summary>
/// Checks if project has outputs for the given parameters hash.
/// NOTE: it checks presence of `parameters.json` only.
/// </summary>
private static async Task <bool> IsGenerated(Project project, OssBucket bucket, string hash)
{
var ossNames = project.OssNameProvider(hash);
return(await bucket.ObjectExistsAsync(ossNames.Parameters));
}
/// <summary>
/// Checks if project has outputs for the given parameters hash.
/// NOTE: it checks presence of `parameters.json` only.
/// </summary>
private static async Task <bool> IsGenerated(OssBucket bucket, OSSObjectNameProvider ossNames)
{
return(await bucket.ObjectExistsAsync(ossNames.Parameters));
}
private async Task ScanBucket(List <MigrationJob> adoptJobs, List <MigrationJob> configJobs, string bucketKeyOld)
{
_logger.LogInformation($"Scanning bucket {bucketKeyOld}");
MigrationBucketKeyProvider bucketProvider;
ProjectService projectService;
using (var scope = _serviceScopeFactory.CreateScope())
{
bucketProvider = scope.ServiceProvider.GetService <MigrationBucketKeyProvider>();
projectService = scope.ServiceProvider.GetService <ProjectService>();
}
OssBucket bucketOld = _bucketFactory.CreateBucket(bucketKeyOld);
OssBucket bucketNew = _bucketFactory.CreateBucket(bucketProvider.GetBucketKeyFromOld(bucketKeyOld));
List <string> projectNamesNew = new List <string>();
try
{
List <string> projectNamesNewFromOss = (List <string>) await projectService.GetProjectNamesAsync(bucketNew);
foreach (string projectName in projectNamesNewFromOss)
{
var ossAttributes = new OssAttributes(projectName);
string metadataFile = ossAttributes.Metadata;
// if metadata file is missing for project we consider that project not migrated
if (await bucketNew.ObjectExistsAsync(metadataFile))
{
projectNamesNew.Add(projectName);
}
}
}
catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound)
{
// swallow non existing item
}
// gather list of cache paramters files from the new bucket
List <string> configKeysNew = new List <string>();
try {
List <ObjectDetails> configODsNew = await bucketNew.GetObjectsAsync($"{ONC.CacheFolder}/");
foreach (ObjectDetails configODNew in configODsNew)
{
if (configODNew.ObjectKey.EndsWith(WebApplication.Utilities.LocalName.Parameters))
{
configKeysNew.Add(configODNew.ObjectKey);
}
}
}
catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound)
{
// swallow non existing item
}
// gather projects to migrate
List <string> projectNamesOld = (List <string>) await projectService.GetProjectNamesAsync(bucketOld);
foreach (string projectName in projectNamesOld)
{
if (!projectNamesNew.Contains(projectName))
{
// new project list does not contain old project => lets copy and adopt
var ossAttributes = new OssAttributes(projectName);
string metadataFile = ossAttributes.Metadata;
try {
ProjectMetadata projectMetadata = await bucketOld.DeserializeAsync <ProjectMetadata>(metadataFile);
ProjectInfo projectInfo = new ProjectInfo();
projectInfo.Name = projectName;
projectInfo.TopLevelAssembly = projectMetadata.TLA;
MigrationJob migrationJob;
using (var scope = _serviceScopeFactory.CreateScope())
{
migrationJob = scope.ServiceProvider.GetService <MigrationJob>();
}
migrationJob.SetJob(JobType.CopyAndAdopt, bucketOld, projectInfo, ONC.ProjectUrl(projectName));
adoptJobs.Add(migrationJob);
} catch (Exception e)
{
_logger.LogError(e, $"Project {projectName} in bucket {bucketKeyOld} does not have metadata file. Skipping it.");
}
}
// process cached configurations
List <ObjectDetails> configODs = await bucketOld.GetObjectsAsync($"{ONC.CacheFolder}/{projectName}/");
foreach (ObjectDetails configOD in configODs)
{
string configKey = configOD.ObjectKey;
if (configKey.EndsWith(WebApplication.Utilities.LocalName.Parameters))
{
// calculate parameter hash based on new algorithmes
InventorParameters parameters = await bucketOld.DeserializeAsync <InventorParameters>(configKey);
string newHash = Crypto.GenerateParametersHashString(parameters);
OSSObjectNameProvider onp = new OSSObjectNameProvider(projectName, newHash);
configKey = onp.Parameters;
if (!configKeysNew.Contains(configKey))
{
ProjectInfo projectInfo = new ProjectInfo();
projectInfo.Name = projectName;
MigrationJob migrationJob;
using (var scope = _serviceScopeFactory.CreateScope())
{
migrationJob = scope.ServiceProvider.GetService <MigrationJob>();
}
migrationJob.SetJob(JobType.GenerateConfiguration, bucketOld, projectInfo, ONC.ProjectUrl(projectName), parameters);
configJobs.Add(migrationJob);
}
}
}
}
// check if any of migrated projects were deleted in old bucket
// (user deleted them after migration started)
foreach (string projectName in projectNamesNew)
{
if (!projectNamesOld.Contains(projectName))
{
MigrationJob migrationJob;
using (var scope = _serviceScopeFactory.CreateScope())
{
migrationJob = scope.ServiceProvider.GetService <MigrationJob>();
}
migrationJob.SetJob(JobType.RemoveNew, bucketNew, new ProjectInfo(projectName), null);
adoptJobs.Add(migrationJob);
}
}
}
/// <summary>
/// Get list of project names for a bucket.
/// </summary>
private async Task <ICollection <string> > GetProjectNamesAsync(OssBucket bucket)
{
return((await bucket.GetObjectsAsync(ONC.ProjectsMask))
.Select(objDetails => ONC.ToProjectName(objDetails.ObjectKey))
.ToList());
}
