/// <summary>
/// Get list of project names for a bucket.
/// </summary>
public async Task <ICollection <string> > GetProjectNamesAsync(OssBucket bucket = null)
{
bucket ??= await _userResolver.GetBucketAsync(true);
var objectDetails = (await bucket.GetObjectsAsync(ONC.ProjectsMask));
var projectNames = objectDetails
.Select(objDetails => ONC.ToProjectName(objDetails.ObjectKey))
.ToList();
return(projectNames);
}
public async Task <StatusCodeResult> DeleteProjects([FromBody] List <string> projectNameList)
{
if (!_userResolver.IsAuthenticated)
{
_logger.LogError("Attempt to delete projects for anonymous user");
return(BadRequest());
}
var bucket = await _userResolver.GetBucketAsync(true);
// collect all oss objects for all provided projects
var tasks = new List <Task>();
foreach (var projectName in projectNameList)
{
tasks.Add(bucket.DeleteObjectAsync(Project.ExactOssName(projectName)));
foreach (var searchMask in ONC.ProjectFileMasks(projectName))
{
var objects = await bucket.GetObjectsAsync(searchMask);
foreach (var objectDetail in objects)
{
tasks.Add(bucket.DeleteObjectAsync(objectDetail.ObjectKey));
}
}
}
// delete the OSS objects
await Task.WhenAll(tasks);
for (var i = 0; i < tasks.Count; i++)
{
if (tasks[i].IsFaulted)
{
_logger.LogError($"Failed to delete file #{i}");
}
}
// delete local cache for all provided projects
foreach (var projectName in projectNameList)
{
var projectStorage = await _userResolver.GetProjectStorageAsync(projectName, ensureDir : false);
projectStorage.DeleteLocal();
}
return(NoContent());
}
public async Task DeleteProjects(ICollection <string> projectNameList, OssBucket bucket = null)
{
bucket ??= await _userResolver.GetBucketAsync(true);
_logger.LogInformation($"deleting projects [{string.Join(", ", projectNameList)}] from bucket {bucket.BucketKey}");
// collect all oss objects for all provided projects
var tasks = new List <Task>();
foreach (var projectName in projectNameList)
{
tasks.Add(bucket.DeleteObjectAsync(Project.ExactOssName(projectName)));
foreach (var searchMask in ONC.ProjectFileMasks(projectName))
{
var objects = await bucket.GetObjectsAsync(searchMask);
foreach (var objectDetail in objects)
{
tasks.Add(bucket.DeleteObjectAsync(objectDetail.ObjectKey));
}
}
}
// delete the OSS objects
await Task.WhenAll(tasks);
for (var i = 0; i < tasks.Count; i++)
{
if (tasks[i].IsFaulted)
{
_logger.LogError($"Failed to delete file #{i}");
}
}
// delete local cache for all provided projects
foreach (var projectName in projectNameList)
{
var projectStorage = await _userResolver.GetProjectStorageAsync(projectName, ensureDir : false);
projectStorage.DeleteLocal();
}
}
public static string ExactOssName(string projectName) => ONC.Join(ONC.ProjectsFolder, projectName);
private async Task ScanBucket(List <MigrationJob> migrationJobs, string bucketKeyOld)
{
OssBucket bucketOld = _bucketFactory.CreateBucket(bucketKeyOld);
OssBucket bucketNew = _bucketFactory.CreateBucket(_bucketProvider.GetBucketKeyFromOld(bucketKeyOld));
List <string> projectNames = (List <string>) await _projectService.GetProjectNamesAsync(bucketOld);
foreach (string projectName in projectNames)
{
var ossAttributes = new OssAttributes(projectName);
string attributeFile = ossAttributes.Metadata;
// check attributes file existance in new destination bucket
if (await bucketNew.ObjectExistsAsync(attributeFile))
{
continue;
}
ProjectMetadata projectMetadata = await bucketOld.DeserializeAsync <ProjectMetadata>(attributeFile);
ProjectInfo projectInfo = new ProjectInfo();
projectInfo.Name = projectName;
projectInfo.TopLevelAssembly = projectMetadata.TLA;
migrationJobs.Add(new MigrationJob(JobType.CopyAndAdopt, bucketOld, projectInfo, ONC.ProjectUrl(projectName)));
}
}
private async Task ScanBucket(List <MigrationJob> adoptJobs, List <MigrationJob> configJobs, string bucketKeyOld)
{
_logger.LogInformation($"Scanning bucket {bucketKeyOld}");
MigrationBucketKeyProvider bucketProvider;
ProjectService projectService;
using (var scope = _serviceScopeFactory.CreateScope())
{
bucketProvider = scope.ServiceProvider.GetService <MigrationBucketKeyProvider>();
projectService = scope.ServiceProvider.GetService <ProjectService>();
}
OssBucket bucketOld = _bucketFactory.CreateBucket(bucketKeyOld);
OssBucket bucketNew = _bucketFactory.CreateBucket(bucketProvider.GetBucketKeyFromOld(bucketKeyOld));
List <string> projectNamesNew = new List <string>();
try
{
List <string> projectNamesNewFromOss = (List <string>) await projectService.GetProjectNamesAsync(bucketNew);
foreach (string projectName in projectNamesNewFromOss)
{
var ossAttributes = new OssAttributes(projectName);
string metadataFile = ossAttributes.Metadata;
// if metadata file is missing for project we consider that project not migrated
if (await bucketNew.ObjectExistsAsync(metadataFile))
{
projectNamesNew.Add(projectName);
}
}
}
catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound)
{
// swallow non existing item
}
// gather list of cache paramters files from the new bucket
List <string> configKeysNew = new List <string>();
try {
List <ObjectDetails> configODsNew = await bucketNew.GetObjectsAsync($"{ONC.CacheFolder}/");
foreach (ObjectDetails configODNew in configODsNew)
{
if (configODNew.ObjectKey.EndsWith(WebApplication.Utilities.LocalName.Parameters))
{
configKeysNew.Add(configODNew.ObjectKey);
}
}
}
catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound)
{
// swallow non existing item
}
// gather projects to migrate
List <string> projectNamesOld = (List <string>) await projectService.GetProjectNamesAsync(bucketOld);
foreach (string projectName in projectNamesOld)
{
if (!projectNamesNew.Contains(projectName))
{
// new project list does not contain old project => lets copy and adopt
var ossAttributes = new OssAttributes(projectName);
string metadataFile = ossAttributes.Metadata;
try {
ProjectMetadata projectMetadata = await bucketOld.DeserializeAsync <ProjectMetadata>(metadataFile);
ProjectInfo projectInfo = new ProjectInfo();
projectInfo.Name = projectName;
projectInfo.TopLevelAssembly = projectMetadata.TLA;
MigrationJob migrationJob;
using (var scope = _serviceScopeFactory.CreateScope())
{
migrationJob = scope.ServiceProvider.GetService <MigrationJob>();
}
migrationJob.SetJob(JobType.CopyAndAdopt, bucketOld, projectInfo, ONC.ProjectUrl(projectName));
adoptJobs.Add(migrationJob);
} catch (Exception e)
{
_logger.LogError(e, $"Project {projectName} in bucket {bucketKeyOld} does not have metadata file. Skipping it.");
}
}
// process cached configurations
List <ObjectDetails> configODs = await bucketOld.GetObjectsAsync($"{ONC.CacheFolder}/{projectName}/");
foreach (ObjectDetails configOD in configODs)
{
string configKey = configOD.ObjectKey;
if (configKey.EndsWith(WebApplication.Utilities.LocalName.Parameters))
{
// calculate parameter hash based on new algorithmes
InventorParameters parameters = await bucketOld.DeserializeAsync <InventorParameters>(configKey);
string newHash = Crypto.GenerateParametersHashString(parameters);
OSSObjectNameProvider onp = new OSSObjectNameProvider(projectName, newHash);
configKey = onp.Parameters;
if (!configKeysNew.Contains(configKey))
{
ProjectInfo projectInfo = new ProjectInfo();
projectInfo.Name = projectName;
MigrationJob migrationJob;
using (var scope = _serviceScopeFactory.CreateScope())
{
migrationJob = scope.ServiceProvider.GetService <MigrationJob>();
}
migrationJob.SetJob(JobType.GenerateConfiguration, bucketOld, projectInfo, ONC.ProjectUrl(projectName), parameters);
configJobs.Add(migrationJob);
}
}
}
}
// check if any of migrated projects were deleted in old bucket
// (user deleted them after migration started)
foreach (string projectName in projectNamesNew)
{
if (!projectNamesOld.Contains(projectName))
{
MigrationJob migrationJob;
using (var scope = _serviceScopeFactory.CreateScope())
{
migrationJob = scope.ServiceProvider.GetService <MigrationJob>();
}
migrationJob.SetJob(JobType.RemoveNew, bucketNew, new ProjectInfo(projectName), null);
adoptJobs.Add(migrationJob);
}
}
}
private async Task ScanBucket(List <MigrationJob> migrationJobs, string bucketKeyOld)
{
OssBucket bucketOld = _bucketFactory.CreateBucket(bucketKeyOld);
OssBucket bucketNew = _bucketFactory.CreateBucket(_bucketProvider.GetBucketKeyFromOld(bucketKeyOld));
List <string> projectNamesNew = new List <string>();
try
{
List <string> projectNamesNewFromOss = (List <string>) await _projectService.GetProjectNamesAsync(bucketNew);
foreach (string projectName in projectNamesNewFromOss)
{
var ossAttributes = new OssAttributes(projectName);
string metadataFile = ossAttributes.Metadata;
// if metadata file is missing for project we consider that project not migrated
if (await bucketNew.ObjectExistsAsync(metadataFile))
{
projectNamesNew.Add(projectName);
}
}
}
catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound)
{
// swallow non existing item
}
List <string> projectNamesOld = (List <string>) await _projectService.GetProjectNamesAsync(bucketOld);
foreach (string projectName in projectNamesOld)
{
if (!projectNamesNew.Contains(projectName))
{
// new project list does not contain old project => lets copy and adopt
var ossAttributes = new OssAttributes(projectName);
string metadataFile = ossAttributes.Metadata;
try {
ProjectMetadata projectMetadata = await bucketOld.DeserializeAsync <ProjectMetadata>(metadataFile);
ProjectInfo projectInfo = new ProjectInfo();
projectInfo.Name = projectName;
projectInfo.TopLevelAssembly = projectMetadata.TLA;
migrationJobs.Add(new MigrationJob(JobType.CopyAndAdopt, bucketOld, projectInfo, ONC.ProjectUrl(projectName)));
} catch (Exception e)
{
_logger.LogError(e, $"Project {projectName} in bucket {bucketKeyOld} does not have metadata file. Skipping that.");
}
}
}
// check if any of migrated projects were deleted in old bucket
// (user deleted them after migration started)
foreach (string projectName in projectNamesNew)
{
if (!projectNamesOld.Contains(projectName))
{
migrationJobs.Add(new MigrationJob(JobType.RemoveNew, bucketNew, new ProjectInfo(projectName), null));
}
}
}
/// <summary>
/// Get list of project names for a bucket.
/// </summary>
private async Task <ICollection <string> > GetProjectNamesAsync(OssBucket bucket)
{
return((await bucket.GetObjectsAsync(ONC.ProjectsMask))
.Select(objDetails => ONC.ToProjectName(objDetails.ObjectKey))
.ToList());
}
public static string ExactOssName(string projectName, string suffix = "") => string.IsNullOrEmpty(suffix) ? ONC.Join(ONC.ProjectsFolder, projectName) : ONC.Join(ONC.ProjectsFolder, projectName, suffix);