public async Task <IEnumerable <ProjectDTO> > ListAsync()
{
var bucket = await _userResolver.GetBucketAsync(tryToCreate : true);
var projectDTOs = new List <ProjectDTO>();
foreach (var projectName in await GetProjectNamesAsync(bucket))
{
// TODO: in future bad projects should not affect project listing. It's a workaround
try
{
ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(projectName); // TODO: expensive to do it in the loop
// handle situation when project is not cached locally
await projectStorage.EnsureAttributesAsync(bucket, ensureDir : true);
projectDTOs.Add(_dtoGenerator.ToDTO(projectStorage));
}
catch (Exception e)
{
// log, swallow and continue (see the comment above)
_logger.LogWarning(e, $"Ignoring '{projectName}' project, which (seems) failed to adopt.");
}
}
return(projectDTOs);
}
/// <summary>
/// Adapt the project.
/// </summary>
public async Task <FdaStatsDTO> AdoptAsync(ProjectInfo projectInfo, string inputDocUrl)
{
_logger.LogInformation($"Adopt project '{projectInfo.Name}'");
var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name);
var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly);
ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData);
if (!result.Success)
{
var message = $"Failed to process '{projectInfo.Name}' project.";
_logger.LogError(message);
throw new FdaProcessingException(message, result.ReportUrl);
}
// rearrange generated data according to the parameters hash
await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly);
_logger.LogInformation("Cache the project locally");
var bucket = await _userResolver.GetBucketAsync();
await projectStorage.EnsureLocalAsync(bucket);
// save adoption statistics
var ossNames = projectStorage.GetOssNames();
await bucket.UploadAsJsonAsync(ossNames.StatsAdopt, result.Stats);
await bucket.CopyAsync(ossNames.StatsAdopt, ossNames.StatsUpdate);
return(FdaStatsDTO.All(result.Stats));
}
/// <summary>
/// Adopt the project.
/// </summary>
public async Task <(FdaStatsDTO stats, string reportUrl)> AdoptAsync(ProjectInfo projectInfo, string inputDocUrl)
{
_logger.LogInformation($"Adopt project '{projectInfo.Name}'");
var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name);
var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly);
ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData);
if (!result.Success)
{
var message = $"Failed to process '{projectInfo.Name}' project.";
_logger.LogError(message);
throw new FdaProcessingException(message, result.ReportUrl);
}
// rearrange generated data according to the parameters hash
await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly);
_logger.LogInformation("Cache the project locally");
var bucket = await _userResolver.GetBucketAsync();
// check for adoption errors
// TECHDEBT: this should be done before `MoveProjectAsync`, but it will leave "garbage" at OSS. Solve it someday.
var messages = await bucket.DeserializeAsync <Message[]>(projectStorage.Project.OssAttributes.AdoptMessages);
var errors = messages.Where(m => m.Severity == Severity.Error).Select(m => m.Text).ToArray();
if (errors.Length > 0)
{
throw new ProcessingException("Adoption failed", errors);
}
await projectStorage.EnsureLocalAsync(bucket);
// save adoption statistics
var ossNames = projectStorage.GetOssNames();
await bucket.UploadAsJsonAsync(ossNames.StatsAdopt, result.Stats);
await bucket.CopyAsync(ossNames.StatsAdopt, ossNames.StatsUpdate);
return(FdaStatsDTO.All(result.Stats), result.ReportUrl);
}
public override async Task ProcessJobAsync(IResultSender resultSender)
{
using var scope = Logger.BeginScope("Project Adoption ({Id})");
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} started.");
// Check for valid project and root names (where applicable)
if ((!string.IsNullOrEmpty(_projectInfo.TopLevelAssembly) && Regex.Match(_projectInfo.TopLevelAssembly, @"[\uFFF0-\uFFFF]").Success) ||
Regex.Match(_projectInfo.Name, @"[\uFFF0-\uFFFF]").Success)
{
Logger.LogInformation($"Replacement charcters found in project name or top level assembly name for job {Id}.");
throw new ProcessingException("Project name or assembly contains unsupported characters",
new[] { "Please refer to https://github.com/Autodesk-Forge/forge-configurator-inventor/blob/master/README.md#project-file-zip-encoding" });
}
// upload the file to OSS
var bucket = await _userResolver.GetBucketAsync(tryToCreate : true);
ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(_projectInfo.Name);
string ossSourceModel = projectStorage.Project.OSSSourceModel;
await bucket.SmartUploadAsync(_fileName, ossSourceModel);
// cleanup before adoption
File.Delete(_fileName);
// adopt the project
bool adopted = false;
FdaStatsDTO stats;
string reportUrl = null;
try
{
string signedUploadedUrl = await bucket.CreateSignedUrlAsync(ossSourceModel);
(stats, reportUrl) = await ProjectWork.AdoptAsync(_projectInfo, signedUploadedUrl);
adopted = true;
}
finally
{
// on any failure during adoption we consider that project adoption failed and it's not usable
if (!adopted)
{
Logger.LogInformation($"Adoption failed. Removing '{ossSourceModel}' OSS object.");
await bucket.DeleteObjectAsync(ossSourceModel);
}
}
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} completed.");
await resultSender.SendSuccessAsync(_dtoGenerator.ToDTO(projectStorage), stats, reportUrl);
}
/// <summary>
/// https://jira.autodesk.com/browse/INVGEN-45256
/// </summary>
/// <param name="payload">project configuration with parameters</param>
/// <returns>project storage</returns>
public async Task <ProjectDTO> AdoptProjectWithParametersAsync(AdoptProjectWithParametersPayload payload)
{
if (!await DoesProjectAlreadyExistAsync(payload.Name))
{
var bucket = await _userResolver.GetBucketAsync();
var signedUrl = await TransferProjectToOssAsync(bucket, payload);
await _projectWork.AdoptAsync(payload, signedUrl);
}
else
{
_logger.LogInformation($"project with name {payload.Name} already exists");
}
var updateDto = (await _projectWork.DoSmartUpdateAsync(payload.Config, payload.Name)).dto;
// use update hash for projectDto generation
var projectDto = _dtoGenerator.ToDTO(await _userResolver.GetProjectStorageAsync(payload.Name), updateDto.Hash);
return(projectDto);
}
public override async Task ProcessJobAsync(IResultSender resultSender)
{
using var scope = Logger.BeginScope("Project Adoption ({Id})");
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} started.");
// upload the file to OSS
var bucket = await _userResolver.GetBucketAsync(tryToCreate : true);
ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(_projectInfo.Name);
string ossSourceModel = projectStorage.Project.OSSSourceModel;
await bucket.SmartUploadAsync(_fileName, ossSourceModel);
// cleanup before adoption
File.Delete(_fileName);
// adopt the project
bool adopted = false;
FdaStatsDTO stats;
try
{
string signedUploadedUrl = await bucket.CreateSignedUrlAsync(ossSourceModel);
stats = await ProjectWork.AdoptAsync(_projectInfo, signedUploadedUrl);
adopted = true;
}
catch (FdaProcessingException fpe)
{
await resultSender.SendErrorAsync(Id, fpe.ReportUrl);
return;
}
finally
{
// on any failure during adoption we consider that project adoption failed and it's not usable
if (!adopted)
{
Logger.LogInformation($"Adoption failed. Removing '{ossSourceModel}' OSS object.");
await bucket.DeleteObjectAsync(ossSourceModel);
}
}
Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} completed.");
await resultSender.SendSuccessAsync(_dtoGenerator.ToDTO(projectStorage), stats);
}
/// <summary>
/// Adapt the project.
/// </summary>
public async Task AdoptAsync(ProjectInfo projectInfo, string inputDocUrl)
{
_logger.LogInformation("Adopt the project");
var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name);
var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly);
ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData);
if (!result.Success)
{
_logger.LogError($"Failed to process '{projectInfo.Name}' project.");
throw new FdaProcessingException($"Failed to process '{projectInfo.Name}' project.", result.ReportUrl);
}
// rearrange generated data according to the parameters hash
await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly);
_logger.LogInformation("Cache the project locally");
var bucket = await _userResolver.GetBucketAsync();
await projectStorage.EnsureLocalAsync(bucket);
}
private async Task <RedirectResult> RedirectToOssObject(string projectName, string hash, Func <OSSObjectNameProvider, bool, string> nameExtractor)
{
ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(projectName);
var ossNameProvider = projectStorage.GetOssNames(hash);
string ossObjectName = nameExtractor(ossNameProvider, projectStorage.IsAssembly);
_logger.LogInformation($"Downloading '{ossObjectName}'");
var bucket = await _userResolver.GetBucketAsync();
var url = await bucket.CreateSignedUrlAsync(ossObjectName);
return(Redirect(url));
}
public async Task InvokeAsync(HttpContext context, UserResolver userResolver, ILogger <SvfRestore> logger)
{
var httpRequest = context.Request;
while (true)
{
// the expected path is like "/data/4EC4EC1C4C0082AB28582C8A50FFC2BF33E42356/Wrench/0B81352BCE7C9CEB8C8EAA7297A8AB64274C75A5/SVF/bubble.json"
// 0 - 'root' for static files (data)
// 1 - User dir (4EC4EC1C4C0082AB28582C8A50FFC2BF33E42356
// 2 - Project ID (Wrench)
// 3 - Parameters hash (0B81352BCE7C9CEB8C8EAA7297A8AB64274C75A5)
// 4 - Subdir for SVF structure (SVF)
// 5 - Manifest file for SVF (bubble.json)
string[] pieces = httpRequest.Path.Value.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (pieces.Length != 6)
{
break;
}
string projectName = pieces[2];
string hash = pieces[3];
var projectStorage = await userResolver.GetProjectStorageAsync(projectName);
// check if SVF dir already exists
var svfDir = projectStorage.GetLocalNames(hash).SvfDir;
if (Directory.Exists(svfDir))
{
break;
}
logger.LogInformation($"Restoring '{hash}' for '{projectName}'");
// download and extract SVF
var bucket = await userResolver.GetBucketAsync();
await projectStorage.EnsureSvfAsync(bucket, hash);
break;
}
// Call the next delegate/middleware in the pipeline
await _next(context);
}
public async Task DeleteProjects(ICollection <string> projectNameList, OssBucket bucket = null)
{
bucket ??= await _userResolver.GetBucketAsync(true);
_logger.LogInformation($"deleting projects [{string.Join(", ", projectNameList)}] from bucket {bucket.BucketKey}");
// collect all oss objects for all provided projects
var tasks = new List <Task>();
foreach (var projectName in projectNameList)
{
tasks.Add(bucket.DeleteObjectAsync(Project.ExactOssName(projectName)));
foreach (var searchMask in ONC.ProjectFileMasks(projectName))
{
var objects = await bucket.GetObjectsAsync(searchMask);
foreach (var objectDetail in objects)
{
tasks.Add(bucket.DeleteObjectAsync(objectDetail.ObjectKey));
}
}
}
// delete the OSS objects
await Task.WhenAll(tasks);
for (var i = 0; i < tasks.Count; i++)
{
if (tasks[i].IsFaulted)
{
_logger.LogError($"Failed to delete file #{i}");
}
}
// delete local cache for all provided projects
foreach (var projectName in projectNameList)
{
var projectStorage = await _userResolver.GetProjectStorageAsync(projectName, ensureDir : false);
projectStorage.DeleteLocal();
}
}
private async Task GenerateConfiguration(string logId)
{
_bucketProvider.SetBucketKeyFromOld(bucket.BucketKey);
_logger.LogInformation($"{logId}: Generating config for project {projectInfo.Name} in bucket {bucket.BucketKey}");
OssBucket bucketNew = await _userResolver.GetBucketAsync();
var storage = await _userResolver.GetProjectStorageAsync(projectInfo.Name);
await storage.EnsureAttributesAsync(bucketNew);
try
{
await _projectWork.DoSmartUpdateAsync(parameters, projectInfo.Name);
_logger.LogInformation($"{logId}: Configuration {parameters} for project {projectInfo.Name} was generated.");
}
catch (Exception e)
{
_logger.LogError(e, $"{logId}: Configuration {parameters} for project {projectInfo.Name} was NOT generated.");
}
return;
}
