private async Task CopyAndAdopt(string logId)
{
_bucketProvider.SetBucketKeyFromOld(bucket.BucketKey);
_logger.LogInformation($"{logId}: Processing new project {projectInfo.Name} in bucket {bucket.BucketKey}");
OssBucket bucketNew = await _userResolver.GetBucketAsync(true);
string signedUrlOld = await bucket.CreateSignedUrlAsync(projectUrl, ObjectAccess.Read);
string signedUrlNew = await bucketNew.CreateSignedUrlAsync(projectUrl, ObjectAccess.ReadWrite);
try
{
await _projectWork.FileTransferAsync(signedUrlOld, signedUrlNew);
}
catch (Exception e)
{
_logger.LogError(e, $"{logId}: Project {projectInfo.Name} cannot be copied.");
return;
}
try
{
await _projectWork.AdoptAsync(projectInfo, signedUrlNew);
_logger.LogInformation($"{logId}: Project {projectInfo.Name} was adopted");
}
catch (Exception e)
{
_logger.LogError(e, $"{logId}: Project {projectInfo.Name} was not adopted");
}
}
public async Task Migrate(List <MigrationJob> migrationJobs)
{
foreach (MigrationJob job in migrationJobs)
{
_bucketProvider.SetBucketKeyFromOld(job.bucketOld.BucketKey);
OssBucket bucketNew = await _userResolver.GetBucketAsync(true);
string signedUrlOld = await job.bucketOld.CreateSignedUrlAsync(job.projectUrl, ObjectAccess.Read);
string signedUrlNew = await bucketNew.CreateSignedUrlAsync(job.projectUrl, ObjectAccess.ReadWrite);
try
{
await _projectWork.FileTransferAsync(signedUrlOld, signedUrlNew);
}
catch (Exception e)
{
_logger.LogError("Project " + job.projectInfo.Name + " cannot be copied\nException:" + e.Message);
continue;
}
try
{
await _projectWork.AdoptAsync(job.projectInfo, signedUrlNew);
_logger.LogInformation("Project " + job.projectInfo.Name + " was adopted");
}
catch (Exception e)
{
_logger.LogError("Project " + job.projectInfo.Name + " was not adopted\nException:" + e.Message);
}
}
}
public async Task InitializeAsync()
{
using var scope = _logger.BeginScope("Init");
_logger.LogInformation("Initializing base data");
// OSS bucket might fail to create, so repeat attempts
var createBucketPolicy = Policy
.Handle <ApiException>()
.WaitAndRetryAsync(
retryCount: 4,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan) => _logger.LogWarning("Cannot create OSS bucket. Repeating")
);
await Task.WhenAll(
// create bundles and activities
_fdaClient.InitializeAsync(),
// create the bucket
createBucketPolicy.ExecuteAsync(async() => await _bucket.CreateAsync())
);
_logger.LogInformation($"Bucket {_bucket.BucketKey} created");
// publish default project files (specified by the appsettings.json)
foreach (DefaultProjectConfiguration defaultProjectConfig in _defaultProjectsConfiguration.Projects)
{
var signedUrl = await _projectService.TransferProjectToOssAsync(_bucket, defaultProjectConfig);
await _projectWork.AdoptAsync(defaultProjectConfig, signedUrl);
}
_logger.LogInformation("Added default projects.");
}
public override async Task ProcessJobAsync(IResultSender resultSender)
{
using var scope = Logger.BeginScope("Project Adoption ({Id})");
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} started.");
// Check for valid project and root names (where applicable)
if ((!string.IsNullOrEmpty(_projectInfo.TopLevelAssembly) && Regex.Match(_projectInfo.TopLevelAssembly, @"[\uFFF0-\uFFFF]").Success) ||
Regex.Match(_projectInfo.Name, @"[\uFFF0-\uFFFF]").Success)
{
Logger.LogInformation($"Replacement charcters found in project name or top level assembly name for job {Id}.");
throw new ProcessingException("Project name or assembly contains unsupported characters",
new[] { "Please refer to https://github.com/Autodesk-Forge/forge-configurator-inventor/blob/master/README.md#project-file-zip-encoding" });
}
// upload the file to OSS
var bucket = await _userResolver.GetBucketAsync(tryToCreate : true);
ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(_projectInfo.Name);
string ossSourceModel = projectStorage.Project.OSSSourceModel;
await bucket.SmartUploadAsync(_fileName, ossSourceModel);
// cleanup before adoption
File.Delete(_fileName);
// adopt the project
bool adopted = false;
FdaStatsDTO stats;
string reportUrl = null;
try
{
string signedUploadedUrl = await bucket.CreateSignedUrlAsync(ossSourceModel);
(stats, reportUrl) = await ProjectWork.AdoptAsync(_projectInfo, signedUploadedUrl);
adopted = true;
}
finally
{
// on any failure during adoption we consider that project adoption failed and it's not usable
if (!adopted)
{
Logger.LogInformation($"Adoption failed. Removing '{ossSourceModel}' OSS object.");
await bucket.DeleteObjectAsync(ossSourceModel);
}
}
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} completed.");
await resultSender.SendSuccessAsync(_dtoGenerator.ToDTO(projectStorage), stats, reportUrl);
}
public async Task InitializeAsync()
{
using var scope = _logger.BeginScope("Init");
_logger.LogInformation("Initializing base data");
// OSS bucket might fail to create, so repeat attempts
var createBucketPolicy = Policy
.Handle <ApiException>()
.WaitAndRetryAsync(
retryCount: 4,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan) => _logger.LogWarning("Cannot create OSS bucket. Repeating")
);
await Task.WhenAll(
// create bundles and activities
_fdaClient.InitializeAsync(),
// create the bucket
createBucketPolicy.ExecuteAsync(async() => await _bucket.CreateAsync())
);
_logger.LogInformation($"Bucket {_bucket.BucketKey} created");
// OSS bucket might be not ready yet, so repeat attempts
var waitForBucketPolicy = Policy
.Handle <ApiException>(e => e.ErrorCode == StatusCodes.Status404NotFound)
.WaitAndRetryAsync(
retryCount: 4,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan) => _logger.LogWarning("Cannot get fresh OSS bucket. Repeating")
);
// publish default project files (specified by the appsettings.json)
foreach (DefaultProjectConfiguration defaultProjectConfig in _defaultProjectsConfiguration.Projects)
{
var projectUrl = defaultProjectConfig.Url;
var project = await _userResolver.GetProjectAsync(defaultProjectConfig.Name);
_logger.LogInformation($"Launching 'TransferData' for {projectUrl}");
string signedUrl = await waitForBucketPolicy.ExecuteAsync(async() => await _bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite));
// TransferData from s3 to oss
await _projectWork.FileTransferAsync(projectUrl, signedUrl);
_logger.LogInformation($"'TransferData' for {projectUrl} is done.");
await _projectWork.AdoptAsync(defaultProjectConfig, signedUrl);
}
_logger.LogInformation("Added default projects.");
}
public override async Task ProcessJobAsync(IResultSender resultSender)
{
using var scope = Logger.BeginScope("Project Adoption ({Id})");
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} started.");
// upload the file to OSS
var bucket = await _userResolver.GetBucketAsync(tryToCreate : true);
ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(_projectInfo.Name);
string ossSourceModel = projectStorage.Project.OSSSourceModel;
await bucket.SmartUploadAsync(_fileName, ossSourceModel);
// cleanup before adoption
File.Delete(_fileName);
// adopt the project
bool adopted = false;
FdaStatsDTO stats;
try
{
string signedUploadedUrl = await bucket.CreateSignedUrlAsync(ossSourceModel);
stats = await ProjectWork.AdoptAsync(_projectInfo, signedUploadedUrl);
adopted = true;
}
catch (FdaProcessingException fpe)
{
await resultSender.SendErrorAsync(Id, fpe.ReportUrl);
return;
}
finally
{
// on any failure during adoption we consider that project adoption failed and it's not usable
if (!adopted)
{
Logger.LogInformation($"Adoption failed. Removing '{ossSourceModel}' OSS object.");
await bucket.DeleteObjectAsync(ossSourceModel);
}
}
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} completed.");
await resultSender.SendSuccessAsync(_dtoGenerator.ToDTO(projectStorage), stats);
}
public async Task <string> AdoptDefaultOnly(bool RemoveCached)
{
string returnValue = "";
if (RemoveCached)
{
List <ObjectDetails> ossFiles = await _forgeOSS.GetBucketObjectsAsync(_bucket.BucketKey, "cache/");
foreach (ObjectDetails file in ossFiles)
{
returnValue += "Removing cache file " + file.ObjectKey + "\n";
try
{
await _forgeOSS.DeleteAsync(_bucket.BucketKey, file.ObjectKey);
} catch (Exception e)
{
returnValue += "Removing cache file " + file.ObjectKey + " failed\nException:" + e.Message + "\n";
}
}
}
foreach (DefaultProjectConfiguration defaultProjectConfig in _defaultProjectsConfiguration.Projects)
{
returnValue += "Project " + defaultProjectConfig.Name + " is being adopted\n";
var projectUrl = defaultProjectConfig.Url;
var project = await _userResolver.GetProjectAsync(defaultProjectConfig.Name);
string signedUrl = await _bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite);
try
{
await _projectWork.AdoptAsync(defaultProjectConfig, signedUrl);
returnValue += "Project " + defaultProjectConfig.Name + " was adopted\n";
}
catch (Exception e)
{
returnValue += "Project " + defaultProjectConfig.Name + " was not adopted\nException:" + e.Message + "\n";
}
}
return(returnValue);
}
/// <summary>
/// https://jira.autodesk.com/browse/INVGEN-45256
/// </summary>
/// <param name="payload">project configuration with parameters</param>
/// <returns>project storage</returns>
public async Task <ProjectDTO> AdoptProjectWithParametersAsync(AdoptProjectWithParametersPayload payload)
{
if (!await DoesProjectAlreadyExistAsync(payload.Name))
{
var bucket = await _userResolver.GetBucketAsync();
var signedUrl = await TransferProjectToOssAsync(bucket, payload);
await _projectWork.AdoptAsync(payload, signedUrl);
}
else
{
_logger.LogInformation($"project with name {payload.Name} already exists");
}
var updateDto = (await _projectWork.DoSmartUpdateAsync(payload.Config, payload.Name)).dto;
// use update hash for projectDto generation
var projectDto = _dtoGenerator.ToDTO(await _userResolver.GetProjectStorageAsync(payload.Name), updateDto.Hash);
return(projectDto);
}