private async Task CopyStateAsync(Project project, string hashFrom, string hashTo, bool isAssembly)
{
// see if the dir exists already
LocalNameProvider localTo = project.LocalNameProvider(hashTo);
if (Directory.Exists(localTo.BaseDir))
{
_logger.LogInformation($"Found existing {hashTo}");
}
// copy local file structure
LocalNameProvider localFrom = project.LocalNameProvider(hashFrom);
// SOMEDAY: performance improvement - replace with symlink when it's supported
// by netcore (https://github.com/dotnet/runtime/issues/24271)
FileSystem.CopyDir(localFrom.BaseDir, localTo.BaseDir);
var bucket = await _userResolver.GetBucketAsync();
// copy OSS files
OSSObjectNameProvider ossFrom = project.OssNameProvider(hashFrom);
OSSObjectNameProvider ossTo = project.OssNameProvider(hashTo);
await Task.WhenAll(
bucket.CopyAsync(ossFrom.Parameters, ossTo.Parameters),
bucket.CopyAsync(ossFrom.GetCurrentModel(isAssembly), ossTo.GetCurrentModel(isAssembly)),
bucket.CopyAsync(ossFrom.ModelView, ossTo.ModelView),
bucket.CopyAsync(ossFrom.Bom, ossTo.Bom));
_logger.LogInformation($"Cache the project locally ({hashTo})");
}
/// <summary>
/// Ensure the project viewables are cached locally.
/// </summary>
/// <param name="ossBucket">OSS bucket.</param>
/// <param name="hash">Parameters hash.</param>
/// <param name="ensureDir">Create local dir if necessary.</param>
public async Task EnsureViewablesAsync(OssBucket ossBucket, string hash, bool ensureDir = true)
{
var localNames = GetLocalNames(hash);
// create the "hashed" dir
if (ensureDir)
{
Directory.CreateDirectory(localNames.BaseDir);
}
OSSObjectNameProvider ossNames = GetOssNames(hash);
using var tempFile = new TempFile();
await Task.WhenAll(
ossBucket.DownloadFileAsync(ossNames.ModelView, tempFile.Name),
ossBucket.DownloadFileAsync(ossNames.Parameters, localNames.Parameters),
ossBucket.DownloadFileAsync(ossNames.Bom, localNames.BOM)
);
// extract SVF from the archive
ZipFile.ExtractToDirectory(tempFile.Name, localNames.SvfDir, overwriteFiles: true); // TODO: non-default encoding is not supported
}
/// <summary>
/// Checks if project has outputs for the given parameters hash.
/// NOTE: it checks presence of `parameters.json` only.
/// </summary>
private static async Task <bool> IsGenerated(OssBucket bucket, OSSObjectNameProvider ossNames)
{
return(await bucket.ObjectExistsAsync(ossNames.Parameters));
}
private async Task ScanBucket(List <MigrationJob> adoptJobs, List <MigrationJob> configJobs, string bucketKeyOld)
{
_logger.LogInformation($"Scanning bucket {bucketKeyOld}");
MigrationBucketKeyProvider bucketProvider;
ProjectService projectService;
using (var scope = _serviceScopeFactory.CreateScope())
{
bucketProvider = scope.ServiceProvider.GetService <MigrationBucketKeyProvider>();
projectService = scope.ServiceProvider.GetService <ProjectService>();
}
OssBucket bucketOld = _bucketFactory.CreateBucket(bucketKeyOld);
OssBucket bucketNew = _bucketFactory.CreateBucket(bucketProvider.GetBucketKeyFromOld(bucketKeyOld));
List <string> projectNamesNew = new List <string>();
try
{
List <string> projectNamesNewFromOss = (List <string>) await projectService.GetProjectNamesAsync(bucketNew);
foreach (string projectName in projectNamesNewFromOss)
{
var ossAttributes = new OssAttributes(projectName);
string metadataFile = ossAttributes.Metadata;
// if metadata file is missing for project we consider that project not migrated
if (await bucketNew.ObjectExistsAsync(metadataFile))
{
projectNamesNew.Add(projectName);
}
}
}
catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound)
{
// swallow non existing item
}
// gather list of cache paramters files from the new bucket
List <string> configKeysNew = new List <string>();
try {
List <ObjectDetails> configODsNew = await bucketNew.GetObjectsAsync($"{ONC.CacheFolder}/");
foreach (ObjectDetails configODNew in configODsNew)
{
if (configODNew.ObjectKey.EndsWith(WebApplication.Utilities.LocalName.Parameters))
{
configKeysNew.Add(configODNew.ObjectKey);
}
}
}
catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound)
{
// swallow non existing item
}
// gather projects to migrate
List <string> projectNamesOld = (List <string>) await projectService.GetProjectNamesAsync(bucketOld);
foreach (string projectName in projectNamesOld)
{
if (!projectNamesNew.Contains(projectName))
{
// new project list does not contain old project => lets copy and adopt
var ossAttributes = new OssAttributes(projectName);
string metadataFile = ossAttributes.Metadata;
try {
ProjectMetadata projectMetadata = await bucketOld.DeserializeAsync <ProjectMetadata>(metadataFile);
ProjectInfo projectInfo = new ProjectInfo();
projectInfo.Name = projectName;
projectInfo.TopLevelAssembly = projectMetadata.TLA;
MigrationJob migrationJob;
using (var scope = _serviceScopeFactory.CreateScope())
{
migrationJob = scope.ServiceProvider.GetService <MigrationJob>();
}
migrationJob.SetJob(JobType.CopyAndAdopt, bucketOld, projectInfo, ONC.ProjectUrl(projectName));
adoptJobs.Add(migrationJob);
} catch (Exception e)
{
_logger.LogError(e, $"Project {projectName} in bucket {bucketKeyOld} does not have metadata file. Skipping it.");
}
}
// process cached configurations
List <ObjectDetails> configODs = await bucketOld.GetObjectsAsync($"{ONC.CacheFolder}/{projectName}/");
foreach (ObjectDetails configOD in configODs)
{
string configKey = configOD.ObjectKey;
if (configKey.EndsWith(WebApplication.Utilities.LocalName.Parameters))
{
// calculate parameter hash based on new algorithmes
InventorParameters parameters = await bucketOld.DeserializeAsync <InventorParameters>(configKey);
string newHash = Crypto.GenerateParametersHashString(parameters);
OSSObjectNameProvider onp = new OSSObjectNameProvider(projectName, newHash);
configKey = onp.Parameters;
if (!configKeysNew.Contains(configKey))
{
ProjectInfo projectInfo = new ProjectInfo();
projectInfo.Name = projectName;
MigrationJob migrationJob;
using (var scope = _serviceScopeFactory.CreateScope())
{
migrationJob = scope.ServiceProvider.GetService <MigrationJob>();
}
migrationJob.SetJob(JobType.GenerateConfiguration, bucketOld, projectInfo, ONC.ProjectUrl(projectName), parameters);
configJobs.Add(migrationJob);
}
}
}
}
// check if any of migrated projects were deleted in old bucket
// (user deleted them after migration started)
foreach (string projectName in projectNamesNew)
{
if (!projectNamesOld.Contains(projectName))
{
MigrationJob migrationJob;
using (var scope = _serviceScopeFactory.CreateScope())
{
migrationJob = scope.ServiceProvider.GetService <MigrationJob>();
}
migrationJob.SetJob(JobType.RemoveNew, bucketNew, new ProjectInfo(projectName), null);
adoptJobs.Add(migrationJob);
}
}
}
