public async Task <IEnumerable <ProjectDTO> > ListAsync() { var bucket = await _userResolver.GetBucketAsync(tryToCreate : true); var projectDTOs = new List <ProjectDTO>(); foreach (var projectName in await GetProjectNamesAsync(bucket)) { // TODO: in future bad projects should not affect project listing. It's a workaround try { ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(projectName); // TODO: expensive to do it in the loop // handle situation when project is not cached locally await projectStorage.EnsureAttributesAsync(bucket, ensureDir : true); projectDTOs.Add(_dtoGenerator.ToDTO(projectStorage)); } catch (Exception e) { // log, swallow and continue (see the comment above) _logger.LogWarning(e, $"Ignoring '{projectName}' project, which (seems) failed to adopt."); } } return(projectDTOs); }
/// <summary> /// Adapt the project. /// </summary> public async Task <FdaStatsDTO> AdoptAsync(ProjectInfo projectInfo, string inputDocUrl) { _logger.LogInformation($"Adopt project '{projectInfo.Name}'"); var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name); var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly); ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData); if (!result.Success) { var message = $"Failed to process '{projectInfo.Name}' project."; _logger.LogError(message); throw new FdaProcessingException(message, result.ReportUrl); } // rearrange generated data according to the parameters hash await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly); _logger.LogInformation("Cache the project locally"); var bucket = await _userResolver.GetBucketAsync(); await projectStorage.EnsureLocalAsync(bucket); // save adoption statistics var ossNames = projectStorage.GetOssNames(); await bucket.UploadAsJsonAsync(ossNames.StatsAdopt, result.Stats); await bucket.CopyAsync(ossNames.StatsAdopt, ossNames.StatsUpdate); return(FdaStatsDTO.All(result.Stats)); }
/// <summary> /// Adopt the project. /// </summary> public async Task <(FdaStatsDTO stats, string reportUrl)> AdoptAsync(ProjectInfo projectInfo, string inputDocUrl) { _logger.LogInformation($"Adopt project '{projectInfo.Name}'"); var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name); var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly); ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData); if (!result.Success) { var message = $"Failed to process '{projectInfo.Name}' project."; _logger.LogError(message); throw new FdaProcessingException(message, result.ReportUrl); } // rearrange generated data according to the parameters hash await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly); _logger.LogInformation("Cache the project locally"); var bucket = await _userResolver.GetBucketAsync(); // check for adoption errors // TECHDEBT: this should be done before `MoveProjectAsync`, but it will leave "garbage" at OSS. Solve it someday. var messages = await bucket.DeserializeAsync <Message[]>(projectStorage.Project.OssAttributes.AdoptMessages); var errors = messages.Where(m => m.Severity == Severity.Error).Select(m => m.Text).ToArray(); if (errors.Length > 0) { throw new ProcessingException("Adoption failed", errors); } await projectStorage.EnsureLocalAsync(bucket); // save adoption statistics var ossNames = projectStorage.GetOssNames(); await bucket.UploadAsJsonAsync(ossNames.StatsAdopt, result.Stats); await bucket.CopyAsync(ossNames.StatsAdopt, ossNames.StatsUpdate); return(FdaStatsDTO.All(result.Stats), result.ReportUrl); }
public override async Task ProcessJobAsync(IResultSender resultSender) { using var scope = Logger.BeginScope("Project Adoption ({Id})"); Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} started."); // Check for valid project and root names (where applicable) if ((!string.IsNullOrEmpty(_projectInfo.TopLevelAssembly) && Regex.Match(_projectInfo.TopLevelAssembly, @"[\uFFF0-\uFFFF]").Success) || Regex.Match(_projectInfo.Name, @"[\uFFF0-\uFFFF]").Success) { Logger.LogInformation($"Replacement charcters found in project name or top level assembly name for job {Id}."); throw new ProcessingException("Project name or assembly contains unsupported characters", new[] { "Please refer to https://github.com/Autodesk-Forge/forge-configurator-inventor/blob/master/README.md#project-file-zip-encoding" }); } // upload the file to OSS var bucket = await _userResolver.GetBucketAsync(tryToCreate : true); ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(_projectInfo.Name); string ossSourceModel = projectStorage.Project.OSSSourceModel; await bucket.SmartUploadAsync(_fileName, ossSourceModel); // cleanup before adoption File.Delete(_fileName); // adopt the project bool adopted = false; FdaStatsDTO stats; string reportUrl = null; try { string signedUploadedUrl = await bucket.CreateSignedUrlAsync(ossSourceModel); (stats, reportUrl) = await ProjectWork.AdoptAsync(_projectInfo, signedUploadedUrl); adopted = true; } finally { // on any failure during adoption we consider that project adoption failed and it's not usable if (!adopted) { Logger.LogInformation($"Adoption failed. Removing '{ossSourceModel}' OSS object."); await bucket.DeleteObjectAsync(ossSourceModel); } } Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} completed."); await resultSender.SendSuccessAsync(_dtoGenerator.ToDTO(projectStorage), stats, reportUrl); }
/// <summary> /// https://jira.autodesk.com/browse/INVGEN-45256 /// </summary> /// <param name="payload">project configuration with parameters</param> /// <returns>project storage</returns> public async Task <ProjectDTO> AdoptProjectWithParametersAsync(AdoptProjectWithParametersPayload payload) { if (!await DoesProjectAlreadyExistAsync(payload.Name)) { var bucket = await _userResolver.GetBucketAsync(); var signedUrl = await TransferProjectToOssAsync(bucket, payload); await _projectWork.AdoptAsync(payload, signedUrl); } else { _logger.LogInformation($"project with name {payload.Name} already exists"); } var updateDto = (await _projectWork.DoSmartUpdateAsync(payload.Config, payload.Name)).dto; // use update hash for projectDto generation var projectDto = _dtoGenerator.ToDTO(await _userResolver.GetProjectStorageAsync(payload.Name), updateDto.Hash); return(projectDto); }
public override async Task ProcessJobAsync(IResultSender resultSender) { using var scope = Logger.BeginScope("Project Adoption ({Id})"); Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} started."); // upload the file to OSS var bucket = await _userResolver.GetBucketAsync(tryToCreate : true); ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(_projectInfo.Name); string ossSourceModel = projectStorage.Project.OSSSourceModel; await bucket.SmartUploadAsync(_fileName, ossSourceModel); // cleanup before adoption File.Delete(_fileName); // adopt the project bool adopted = false; FdaStatsDTO stats; try { string signedUploadedUrl = await bucket.CreateSignedUrlAsync(ossSourceModel); stats = await ProjectWork.AdoptAsync(_projectInfo, signedUploadedUrl); adopted = true; } catch (FdaProcessingException fpe) { await resultSender.SendErrorAsync(Id, fpe.ReportUrl); return; } finally { // on any failure during adoption we consider that project adoption failed and it's not usable if (!adopted) { Logger.LogInformation($"Adoption failed. Removing '{ossSourceModel}' OSS object."); await bucket.DeleteObjectAsync(ossSourceModel); } } Logger.LogInformation($"ProcessJob (Adopt) {Id} for project {_projectInfo.Name} completed."); await resultSender.SendSuccessAsync(_dtoGenerator.ToDTO(projectStorage), stats); }
/// <summary> /// Adapt the project. /// </summary> public async Task AdoptAsync(ProjectInfo projectInfo, string inputDocUrl) { _logger.LogInformation("Adopt the project"); var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name); var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly); ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData); if (!result.Success) { _logger.LogError($"Failed to process '{projectInfo.Name}' project."); throw new FdaProcessingException($"Failed to process '{projectInfo.Name}' project.", result.ReportUrl); } // rearrange generated data according to the parameters hash await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly); _logger.LogInformation("Cache the project locally"); var bucket = await _userResolver.GetBucketAsync(); await projectStorage.EnsureLocalAsync(bucket); }
private async Task <RedirectResult> RedirectToOssObject(string projectName, string hash, Func <OSSObjectNameProvider, bool, string> nameExtractor) { ProjectStorage projectStorage = await _userResolver.GetProjectStorageAsync(projectName); var ossNameProvider = projectStorage.GetOssNames(hash); string ossObjectName = nameExtractor(ossNameProvider, projectStorage.IsAssembly); _logger.LogInformation($"Downloading '{ossObjectName}'"); var bucket = await _userResolver.GetBucketAsync(); var url = await bucket.CreateSignedUrlAsync(ossObjectName); return(Redirect(url)); }
public async Task InvokeAsync(HttpContext context, UserResolver userResolver, ILogger <SvfRestore> logger) { var httpRequest = context.Request; while (true) { // the expected path is like "/data/4EC4EC1C4C0082AB28582C8A50FFC2BF33E42356/Wrench/0B81352BCE7C9CEB8C8EAA7297A8AB64274C75A5/SVF/bubble.json" // 0 - 'root' for static files (data) // 1 - User dir (4EC4EC1C4C0082AB28582C8A50FFC2BF33E42356 // 2 - Project ID (Wrench) // 3 - Parameters hash (0B81352BCE7C9CEB8C8EAA7297A8AB64274C75A5) // 4 - Subdir for SVF structure (SVF) // 5 - Manifest file for SVF (bubble.json) string[] pieces = httpRequest.Path.Value.Split('/', StringSplitOptions.RemoveEmptyEntries); if (pieces.Length != 6) { break; } string projectName = pieces[2]; string hash = pieces[3]; var projectStorage = await userResolver.GetProjectStorageAsync(projectName); // check if SVF dir already exists var svfDir = projectStorage.GetLocalNames(hash).SvfDir; if (Directory.Exists(svfDir)) { break; } logger.LogInformation($"Restoring '{hash}' for '{projectName}'"); // download and extract SVF var bucket = await userResolver.GetBucketAsync(); await projectStorage.EnsureSvfAsync(bucket, hash); break; } // Call the next delegate/middleware in the pipeline await _next(context); }
public async Task DeleteProjects(ICollection <string> projectNameList, OssBucket bucket = null) { bucket ??= await _userResolver.GetBucketAsync(true); _logger.LogInformation($"deleting projects [{string.Join(", ", projectNameList)}] from bucket {bucket.BucketKey}"); // collect all oss objects for all provided projects var tasks = new List <Task>(); foreach (var projectName in projectNameList) { tasks.Add(bucket.DeleteObjectAsync(Project.ExactOssName(projectName))); foreach (var searchMask in ONC.ProjectFileMasks(projectName)) { var objects = await bucket.GetObjectsAsync(searchMask); foreach (var objectDetail in objects) { tasks.Add(bucket.DeleteObjectAsync(objectDetail.ObjectKey)); } } } // delete the OSS objects await Task.WhenAll(tasks); for (var i = 0; i < tasks.Count; i++) { if (tasks[i].IsFaulted) { _logger.LogError($"Failed to delete file #{i}"); } } // delete local cache for all provided projects foreach (var projectName in projectNameList) { var projectStorage = await _userResolver.GetProjectStorageAsync(projectName, ensureDir : false); projectStorage.DeleteLocal(); } }
private async Task GenerateConfiguration(string logId) { _bucketProvider.SetBucketKeyFromOld(bucket.BucketKey); _logger.LogInformation($"{logId}: Generating config for project {projectInfo.Name} in bucket {bucket.BucketKey}"); OssBucket bucketNew = await _userResolver.GetBucketAsync(); var storage = await _userResolver.GetProjectStorageAsync(projectInfo.Name); await storage.EnsureAttributesAsync(bucketNew); try { await _projectWork.DoSmartUpdateAsync(parameters, projectInfo.Name); _logger.LogInformation($"{logId}: Configuration {parameters} for project {projectInfo.Name} was generated."); } catch (Exception e) { _logger.LogError(e, $"{logId}: Configuration {parameters} for project {projectInfo.Name} was NOT generated."); } return; }