public async Task <IEnumerable <ProjectDTO> > ListAsync()
{
var bucket = await _userResolver.GetBucketAsync(tryToCreate : true);
var projectDTOs = new List <ProjectDTO>();
foreach (var projectName in await GetProjectNamesAsync(bucket))
{
// TODO: in future bad projects should not affect project listing. It's a workaround
try
{
ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(projectName); // TODO: expensive to do it in the loop
// handle situation when project is not cached locally
await projectStorage.EnsureAttributesAsync(bucket, ensureDir : true);
projectDTOs.Add(_dtoGenerator.ToDTO(projectStorage));
}
catch (Exception e)
{
// log, swallow and continue (see the comment above)
_logger.LogWarning(e, $"Ignoring '{projectName}' project, which (seems) failed to adopt.");
}
}
return(projectDTOs);
}
public async Task <bool> Get()
{
bool result = true;
ApiResponse <dynamic> ossObjectResponse = null;
try
{
var bucket = await _userResolver.GetBucketAsync();
ossObjectResponse = await bucket.GetObjectAsync(ONC.ShowParametersChanged);
}
catch (ApiException ex) when(ex.ErrorCode == 404)
{
// the file is not found. Just swallow the exception
}
if (ossObjectResponse != null)
{
using (Stream objectStream = ossObjectResponse.Data)
{
result = await JsonSerializer.DeserializeAsync <bool>(objectStream);
}
}
return(result);
}
/// <summary>
/// Get list of project names for a bucket.
/// </summary>
public async Task <ICollection <string> > GetProjectNamesAsync(OssBucket bucket = null)
{
bucket ??= await _userResolver.GetBucketAsync(true);
var objectDetails = (await bucket.GetObjectsAsync(ONC.ProjectsMask));
var projectNames = objectDetails
.Select(objDetails => ONC.ToProjectName(objDetails.ObjectKey))
.ToList();
return(projectNames);
}
public async Task Migrate(List <MigrationJob> migrationJobs)
{
foreach (MigrationJob job in migrationJobs)
{
_bucketProvider.SetBucketKeyFromOld(job.bucketOld.BucketKey);
OssBucket bucketNew = await _userResolver.GetBucketAsync(true);
string signedUrlOld = await job.bucketOld.CreateSignedUrlAsync(job.projectUrl, ObjectAccess.Read);
string signedUrlNew = await bucketNew.CreateSignedUrlAsync(job.projectUrl, ObjectAccess.ReadWrite);
try
{
await _projectWork.FileTransferAsync(signedUrlOld, signedUrlNew);
}
catch (Exception e)
{
_logger.LogError("Project " + job.projectInfo.Name + " cannot be copied\nException:" + e.Message);
continue;
}
try
{
await _projectWork.AdoptAsync(job.projectInfo, signedUrlNew);
_logger.LogInformation("Project " + job.projectInfo.Name + " was adopted");
}
catch (Exception e)
{
_logger.LogError("Project " + job.projectInfo.Name + " was not adopted\nException:" + e.Message);
}
}
}
private async Task CopyAndAdopt(MigrationJob job)
{
_bucketProvider.SetBucketKeyFromOld(job.bucket.BucketKey);
OssBucket bucketNew = await _userResolver.GetBucketAsync(true);
string signedUrlOld = await job.bucket.CreateSignedUrlAsync(job.projectUrl, ObjectAccess.Read);
string signedUrlNew = await bucketNew.CreateSignedUrlAsync(job.projectUrl, ObjectAccess.ReadWrite);
try
{
await _projectWork.FileTransferAsync(signedUrlOld, signedUrlNew);
}
catch (Exception e)
{
_logger.LogError(e, $"Project {job.projectInfo.Name} cannot be copied.");
return;
}
try
{
await _projectWork.AdoptAsync(job.projectInfo, signedUrlNew);
_logger.LogInformation($"Project {job.projectInfo.Name} was adopted");
}
catch (Exception e)
{
_logger.LogError(e, $"Project {job.projectInfo.Name} was not adopted");
}
}
/// <summary>
/// Adapt the project.
/// </summary>
public async Task <FdaStatsDTO> AdoptAsync(ProjectInfo projectInfo, string inputDocUrl)
{
_logger.LogInformation($"Adopt project '{projectInfo.Name}'");
var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name);
var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly);
ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData);
if (!result.Success)
{
var message = $"Failed to process '{projectInfo.Name}' project.";
_logger.LogError(message);
throw new FdaProcessingException(message, result.ReportUrl);
}
// rearrange generated data according to the parameters hash
await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly);
_logger.LogInformation("Cache the project locally");
var bucket = await _userResolver.GetBucketAsync();
await projectStorage.EnsureLocalAsync(bucket);
// save adoption statistics
var ossNames = projectStorage.GetOssNames();
await bucket.UploadAsJsonAsync(ossNames.StatsAdopt, result.Stats);
await bucket.CopyAsync(ossNames.StatsAdopt, ossNames.StatsUpdate);
return(FdaStatsDTO.All(result.Stats));
}
public override async Task ProcessJobAsync(IResultSender resultSender)
{
using var scope = Logger.BeginScope("Project Adoption ({Id})");
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} started.");
// Check for valid project and root names (where applicable)
if ((!string.IsNullOrEmpty(_projectInfo.TopLevelAssembly) && Regex.Match(_projectInfo.TopLevelAssembly, @"[\uFFF0-\uFFFF]").Success) ||
Regex.Match(_projectInfo.Name, @"[\uFFF0-\uFFFF]").Success)
{
Logger.LogInformation($"Replacement charcters found in project name or top level assembly name for job {Id}.");
throw new ProcessingException("Project name or assembly contains unsupported characters",
new[] { "Please refer to https://github.com/Autodesk-Forge/forge-configurator-inventor/blob/master/README.md#project-file-zip-encoding" });
}
// upload the file to OSS
var bucket = await _userResolver.GetBucketAsync(tryToCreate : true);
ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(_projectInfo.Name);
string ossSourceModel = projectStorage.Project.OSSSourceModel;
await bucket.SmartUploadAsync(_fileName, ossSourceModel);
// cleanup before adoption
File.Delete(_fileName);
// adopt the project
bool adopted = false;
FdaStatsDTO stats;
string reportUrl = null;
try
{
string signedUploadedUrl = await bucket.CreateSignedUrlAsync(ossSourceModel);
(stats, reportUrl) = await ProjectWork.AdoptAsync(_projectInfo, signedUploadedUrl);
adopted = true;
}
finally
{
// on any failure during adoption we consider that project adoption failed and it's not usable
if (!adopted)
{
Logger.LogInformation($"Adoption failed. Removing '{ossSourceModel}' OSS object.");
await bucket.DeleteObjectAsync(ossSourceModel);
}
}
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} completed.");
await resultSender.SendSuccessAsync(_dtoGenerator.ToDTO(projectStorage), stats, reportUrl);
}
private async Task GenerateConfiguration(string logId)
{
_bucketProvider.SetBucketKeyFromOld(bucket.BucketKey);
_logger.LogInformation($"{logId}: Generating config for project {projectInfo.Name} in bucket {bucket.BucketKey}");
OssBucket bucketNew = await _userResolver.GetBucketAsync();
try
{
await _projectWork.DoSmartUpdateAsync(parameters, projectInfo.Name);
_logger.LogInformation($"{logId}: Configuration {parameters} for project {projectInfo.Name} was generated.");
}
catch (Exception e)
{
_logger.LogError(e, $"{logId}: Configuration {parameters} for project {projectInfo.Name} was NOT generated.");
}
return;
}
/// <summary>
/// https://jira.autodesk.com/browse/INVGEN-45256
/// </summary>
/// <param name="payload">project configuration with parameters</param>
/// <returns>project storage</returns>
public async Task <ProjectDTO> AdoptProjectWithParametersAsync(AdoptProjectWithParametersPayload payload)
{
if (!await DoesProjectAlreadyExistAsync(payload.Name))
{
var bucket = await _userResolver.GetBucketAsync();
var signedUrl = await TransferProjectToOssAsync(bucket, payload);
await _projectWork.AdoptAsync(payload, signedUrl);
}
else
{
_logger.LogInformation($"project with name {payload.Name} already exists");
}
var updateDto = (await _projectWork.DoSmartUpdateAsync(payload.Config, payload.Name)).dto;
// use update hash for projectDto generation
var projectDto = _dtoGenerator.ToDTO(await _userResolver.GetProjectStorageAsync(payload.Name), updateDto.Hash);
return(projectDto);
}
/// <summary>
/// Create adoption data.
/// </summary>
/// <param name="docUrl">URL to the input Inventor document (IPT or zipped IAM)</param>
/// <param name="tlaFilename">Top level assembly in the ZIP. (if any)</param>
public async Task <AdoptionData> ForAdoptionAsync(string docUrl, string tlaFilename)
{
var bucket = await _userResolver.GetBucketAsync();
var urls = await Task.WhenAll(bucket.CreateSignedUrlAsync(Thumbnail, ObjectAccess.Write),
bucket.CreateSignedUrlAsync(SVF, ObjectAccess.Write),
bucket.CreateSignedUrlAsync(Parameters, ObjectAccess.Write),
bucket.CreateSignedUrlAsync(OutputModelIAM, ObjectAccess.Write),
bucket.CreateSignedUrlAsync(OutputModelIPT, ObjectAccess.Write),
bucket.CreateSignedUrlAsync(BomJson, ObjectAccess.Write));
return(new AdoptionData
{
InputDocUrl = docUrl,
ThumbnailUrl = urls[0],
SvfUrl = urls[1],
ParametersJsonUrl = urls[2],
OutputIAMModelUrl = urls[3],
OutputIPTModelUrl = urls[4],
BomUrl = urls[5],
TLA = tlaFilename
});
}
public override async Task ProcessJobAsync(IResultSender resultSender)
{
using var scope = Logger.BeginScope("Project Adoption ({Id})");
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} started.");
// upload the file to OSS
var bucket = await _userResolver.GetBucketAsync(tryToCreate : true);
ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(_projectInfo.Name);
string ossSourceModel = projectStorage.Project.OSSSourceModel;
await bucket.SmartUploadAsync(_fileName, ossSourceModel);
// cleanup before adoption
File.Delete(_fileName);
// adopt the project
bool adopted = false;
FdaStatsDTO stats;
try
{
string signedUploadedUrl = await bucket.CreateSignedUrlAsync(ossSourceModel);
stats = await ProjectWork.AdoptAsync(_projectInfo, signedUploadedUrl);
adopted = true;
}
catch (FdaProcessingException fpe)
{
await resultSender.SendErrorAsync(Id, fpe.ReportUrl);
return;
}
finally
{
// on any failure during adoption we consider that project adoption failed and it's not usable
if (!adopted)
{
Logger.LogInformation($"Adoption failed. Removing '{ossSourceModel}' OSS object.");
await bucket.DeleteObjectAsync(ossSourceModel);
}
}
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} completed.");
await resultSender.SendSuccessAsync(_dtoGenerator.ToDTO(projectStorage), stats);
}
/// <summary>
/// Adapt the project.
/// </summary>
public async Task AdoptAsync(ProjectInfo projectInfo, string inputDocUrl)
{
_logger.LogInformation("Adopt the project");
var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name);
var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly);
ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData);
if (!result.Success)
{
_logger.LogError($"Failed to process '{projectInfo.Name}' project.");
throw new FdaProcessingException($"Failed to process '{projectInfo.Name}' project.", result.ReportUrl);
}
// rearrange generated data according to the parameters hash
await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly);
_logger.LogInformation("Cache the project locally");
var bucket = await _userResolver.GetBucketAsync();
await projectStorage.EnsureLocalAsync(bucket);
}
private async Task <RedirectResult> RedirectToOssObject(string projectName, string hash, Func <OSSObjectNameProvider, bool, string> nameExtractor)
{
ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(projectName);
var ossNameProvider = projectStorage.GetOssNames(hash);
string ossObjectName = nameExtractor(ossNameProvider, projectStorage.IsAssembly);
_logger.LogInformation($"Downloading '{ossObjectName}'");
var bucket = await _userResolver.GetBucketAsync();
var url = await bucket.CreateSignedUrlAsync(ossObjectName);
return(Redirect(url));
}
public async Task InvokeAsync(HttpContext context, UserResolver userResolver, ILogger <SvfRestore> logger)
{
var httpRequest = context.Request;
while (true)
{
// the expected path is like "/data/4EC4EC1C4C0082AB28582C8A50FFC2BF33E42356/Wrench/0B81352BCE7C9CEB8C8EAA7297A8AB64274C75A5/SVF/bubble.json"
// 0 - 'root' for static files (data)
// 1 - User dir (4EC4EC1C4C0082AB28582C8A50FFC2BF33E42356
// 2 - Project ID (Wrench)
// 3 - Parameters hash (0B81352BCE7C9CEB8C8EAA7297A8AB64274C75A5)
// 4 - Subdir for SVF structure (SVF)
// 5 - Manifest file for SVF (bubble.json)
string[] pieces = httpRequest.Path.Value.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (pieces.Length != 6)
{
break;
}
string projectName = pieces[2];
string hash = pieces[3];
var projectStorage = await userResolver.GetProjectStorageAsync(projectName);
// check if SVF dir already exists
var svfDir = projectStorage.GetLocalNames(hash).SvfDir;
if (Directory.Exists(svfDir))
{
break;
}
logger.LogInformation($"Restoring '{hash}' for '{projectName}'");
// download and extract SVF
var bucket = await userResolver.GetBucketAsync();
await projectStorage.EnsureSvfAsync(bucket, hash);
break;
}
// Call the next delegate/middleware in the pipeline
await _next(context);
}
/// <summary>
/// Adopt the project.
/// </summary>
public async Task <(FdaStatsDTO stats, string reportUrl)> AdoptAsync(ProjectInfo projectInfo, string inputDocUrl)
{
_logger.LogInformation($"Adopt project '{projectInfo.Name}'");
var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name);
var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly);
ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData);
if (!result.Success)
{
var message = $"Failed to process '{projectInfo.Name}' project.";
_logger.LogError(message);
throw new FdaProcessingException(message, result.ReportUrl);
}
// rearrange generated data according to the parameters hash
await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly);
_logger.LogInformation("Cache the project locally");
var bucket = await _userResolver.GetBucketAsync();
// check for adoption errors
// TECHDEBT: this should be done before `MoveProjectAsync`, but it will leave "garbage" at OSS. Solve it someday.
var messages = await bucket.DeserializeAsync <Message[]>(projectStorage.Project.OssAttributes.AdoptMessages);
var errors = messages.Where(m => m.Severity == Severity.Error).Select(m => m.Text).ToArray();
if (errors.Length > 0)
{
throw new ProcessingException("Adoption failed", errors);
}
await projectStorage.EnsureLocalAsync(bucket);
// save adoption statistics
var ossNames = projectStorage.GetOssNames();
await bucket.UploadAsJsonAsync(ossNames.StatsAdopt, result.Stats);
await bucket.CopyAsync(ossNames.StatsAdopt, ossNames.StatsUpdate);
return(FdaStatsDTO.All(result.Stats), result.ReportUrl);
}
