/// <summary>
/// Adapt the project.
/// </summary>
public async Task <FdaStatsDTO> AdoptAsync(ProjectInfo projectInfo, string inputDocUrl)
{
_logger.LogInformation($"Adopt project '{projectInfo.Name}'");
var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name);
var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly);
ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData);
if (!result.Success)
{
var message = $"Failed to process '{projectInfo.Name}' project.";
_logger.LogError(message);
throw new FdaProcessingException(message, result.ReportUrl);
}
// rearrange generated data according to the parameters hash
await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly);
_logger.LogInformation("Cache the project locally");
var bucket = await _userResolver.GetBucketAsync();
await projectStorage.EnsureLocalAsync(bucket);
// save adoption statistics
var ossNames = projectStorage.GetOssNames();
await bucket.UploadAsJsonAsync(ossNames.StatsAdopt, result.Stats);
await bucket.CopyAsync(ossNames.StatsAdopt, ossNames.StatsUpdate);
return(FdaStatsDTO.All(result.Stats));
}
public override async Task ProcessJobAsync(IResultSender resultSender)
{
using var scope = Logger.BeginScope("Export Drawing PDF ({Id})");
Logger.LogInformation($"ProcessJob (ExportDrawingPDF) {Id} for project {ProjectId} started.");
FdaStatsDTO stats = await ProjectWork.ExportDrawingPdfAsync(ProjectId, _hash);
Logger.LogInformation($"ProcessJob (ExportDrawingPDF) {Id} for project {ProjectId} completed.");
string url = "";
if (stats != null)
{
url = _linkGenerator.GetPathByAction(controller: "Download",
action: "DrawingPdf",
values: new { projectName = ProjectId, hash = _hash });
// when local url starts with slash, it does not work, because it is doubled in url
if (url.StartsWith('/'))
{
url = url.Substring(1);
}
}
await resultSender.SendSuccessAsync(url, stats);
}
/// <summary>
/// Generate project data for the given parameters and cache results locally.
/// </summary>
/// <returns>Resulting parameters hash</returns>
private async Task <(string hash, FdaStatsDTO stats, string reportUrl)> UpdateAsync(ProjectStorage storage, InventorParameters parameters,
string hash, bool bForceUpdate = false)
{
_logger.LogInformation("Update the project");
var bucket = await _userResolver.GetBucketAsync();
var isUpdateExists = bForceUpdate ? false : await IsGenerated(bucket, storage.GetOssNames(hash));
FdaStatsDTO stats;
string reportUrl;
if (isUpdateExists)
{
_logger.LogInformation("Detected existing outputs at OSS");
var statsNative = await bucket.DeserializeAsync <List <Statistics> >(storage.GetOssNames(hash).StatsUpdate);
stats = FdaStatsDTO.CreditsOnly(statsNative);
reportUrl = null;
}
else
{
Project project = storage.Project;
var inputDocUrl = await bucket.CreateSignedUrlAsync(project.OSSSourceModel);
UpdateData updateData = await _arranger.ForUpdateAsync(inputDocUrl, storage.Metadata.TLA, parameters);
ProcessingResult result = await _fdaClient.UpdateAsync(updateData);
if (!result.Success)
{
_logger.LogError($"Failed to update '{project.Name}' project.");
throw new FdaProcessingException($"Failed to update '{project.Name}' project.", result.ReportUrl);
}
_logger.LogInformation("Moving files around");
// rearrange generated data according to the parameters hash
// NOTE: hash might be changed if Inventor adjust them!
hash = await _arranger.MoveViewablesAsync(project, storage.IsAssembly);
// process statistics
await bucket.UploadAsJsonAsync(storage.GetOssNames(hash).StatsUpdate, result.Stats);
stats = FdaStatsDTO.All(result.Stats);
reportUrl = result.ReportUrl;
}
_logger.LogInformation($"Cache the project locally ({hash})");
// and now cache the generated stuff locally
await storage.EnsureViewablesAsync(bucket, hash);
return(hash, stats, reportUrl);
}
/// <summary>
/// Update project state with the parameters (or take it from cache).
/// </summary>
public async Task <(ProjectStateDTO dto, FdaStatsDTO stats, string reportUrl)> DoSmartUpdateAsync(InventorParameters parameters,
string projectId, bool bForceUpdate = false)
{
var hash = Crypto.GenerateObjectHashString(parameters);
_logger.LogInformation($"Incoming parameters hash is {hash}");
var storage = await _userResolver.GetProjectStorageAsync(projectId);
FdaStatsDTO stats;
var localNames = storage.GetLocalNames(hash);
string reportUrl;
// check if the data cached already
if (Directory.Exists(localNames.SvfDir) && !bForceUpdate)
{
_logger.LogInformation("Found cached data.");
// restore statistics
var bucket = await _userResolver.GetBucketAsync();
var statsNative = await bucket.DeserializeAsync <List <Statistics> >(storage.GetOssNames(hash).StatsUpdate);
stats = FdaStatsDTO.CreditsOnly(statsNative);
reportUrl = null;
}
else
{
string resultingHash;
(resultingHash, stats, reportUrl) = await UpdateAsync(storage, parameters, hash, bForceUpdate);
if (!hash.Equals(resultingHash, StringComparison.Ordinal))
{
_logger.LogInformation($"Update returned different parameters. Hash is {resultingHash}.");
await CopyStateAsync(storage.Project, resultingHash, hash, storage.IsAssembly);
// update
hash = resultingHash;
}
}
var dto = _dtoGenerator.MakeProjectDTO <ProjectStateDTO>(storage, hash);
dto.Parameters = Json.DeserializeFile <InventorParameters>(localNames.Parameters);
return(dto, stats, reportUrl);
}
/// <summary>
/// Generate RFA (or take it from cache).
/// </summary>
public async Task <FdaStatsDTO> GenerateRfaAsync(string projectName, string hash)
{
_logger.LogInformation($"Generating RFA for hash {hash}");
ProjectStorage storage = await _userResolver.GetProjectStorageAsync(projectName);
Project project = storage.Project;
//// *********************************************
//// temporary fail *********************************************
//_logger.LogError($"Failed to generate SAT file");
//throw new FdaProcessingException($"Failed to generate SAT file", "https://localhost:5000/#");
//// *********************************************
var ossNames = project.OssNameProvider(hash);
var bucket = await _userResolver.GetBucketAsync();
// check if RFA file is already generated
if (await bucket.ObjectExistsAsync(ossNames.Rfa))
{
var stats = await bucket.DeserializeAsync <Statistics[]>(ossNames.StatsRFA);
return(FdaStatsDTO.CreditsOnly(stats));
}
// OK, nothing in cache - generate it now
var inputDocUrl = await bucket.CreateSignedUrlAsync(ossNames.GetCurrentModel(storage.IsAssembly));
ProcessingArgs satData = await _arranger.ForSatAsync(inputDocUrl, storage.Metadata.TLA);
ProcessingArgs rfaData = await _arranger.ForRfaAsync(satData.SatUrl);
ProcessingResult result = await _fdaClient.GenerateRfa(satData, rfaData);
if (!result.Success)
{
_logger.LogError($"{result.ErrorMessage} for project {project.Name} and hash {hash}");
throw new FdaProcessingException($"{result.ErrorMessage} for project {project.Name} and hash {hash}", result.ReportUrl);
}
await _arranger.MoveRfaAsync(project, hash);
await bucket.UploadAsJsonAsync(ossNames.StatsRFA, result.Stats);
return(FdaStatsDTO.All(result.Stats));
}
public void Single()
{
var stats = MakeStat(downloadOffset: 15, instructionStartedOffset: 20, instructionEndedOffset: 40, uploadOffset: 50, finishedOffset: 52);
var calculated = FdaStatsDTO.All(new List <Statistics>(new [] { stats }));
Assert.Equal(15, calculated.Queueing.Value, 1);
Assert.Equal(5, calculated.Download.Value, 1);
Assert.Equal(20, calculated.Processing.Value, 1);
Assert.Equal(10, calculated.Upload.Value, 1);
Assert.Equal(52, calculated.Total.Value, 1);
// validate credits
const double paidTimeSec = 35.0; // download + processing + upload
const double expected = paidTimeSec * CostPerHour / 3600;
Assert.Equal(expected, calculated.Credits, 4);
}
/// <summary>
/// Adopt the project.
/// </summary>
public async Task <(FdaStatsDTO stats, string reportUrl)> AdoptAsync(ProjectInfo projectInfo, string inputDocUrl)
{
_logger.LogInformation($"Adopt project '{projectInfo.Name}'");
var projectStorage = await _userResolver.GetProjectStorageAsync(projectInfo.Name);
var adoptionData = await _arranger.ForAdoptionAsync(inputDocUrl, projectInfo.TopLevelAssembly);
ProcessingResult result = await _fdaClient.AdoptAsync(adoptionData);
if (!result.Success)
{
var message = $"Failed to process '{projectInfo.Name}' project.";
_logger.LogError(message);
throw new FdaProcessingException(message, result.ReportUrl);
}
// rearrange generated data according to the parameters hash
await _arranger.MoveProjectAsync(projectStorage.Project, projectInfo.TopLevelAssembly);
_logger.LogInformation("Cache the project locally");
var bucket = await _userResolver.GetBucketAsync();
// check for adoption errors
// TECHDEBT: this should be done before `MoveProjectAsync`, but it will leave "garbage" at OSS. Solve it someday.
var messages = await bucket.DeserializeAsync <Message[]>(projectStorage.Project.OssAttributes.AdoptMessages);
var errors = messages.Where(m => m.Severity == Severity.Error).Select(m => m.Text).ToArray();
if (errors.Length > 0)
{
throw new ProcessingException("Adoption failed", errors);
}
await projectStorage.EnsureLocalAsync(bucket);
// save adoption statistics
var ossNames = projectStorage.GetOssNames();
await bucket.UploadAsJsonAsync(ossNames.StatsAdopt, result.Stats);
await bucket.CopyAsync(ossNames.StatsAdopt, ossNames.StatsUpdate);
return(FdaStatsDTO.All(result.Stats), result.ReportUrl);
}
public override async Task ProcessJobAsync(IResultSender resultSender)
{
using var scope = Logger.BeginScope("Drawing generation ({Id})");
Logger.LogInformation($"ProcessJob (Drawing) {Id} for project {ProjectId} started.");
FdaStatsDTO stats = await ProjectWork.GenerateDrawingAsync(ProjectId, _hash);
Logger.LogInformation($"ProcessJob (Drawing) {Id} for project {ProjectId} completed.");
// TODO: this url can be generated right away... we can simply acknowledge that OSS file is ready,
// without generating URL here
var drawingUrl = _linkGenerator.GetPathByAction(controller: "Download",
action: "Drawing",
values: new { projectName = ProjectId, hash = _hash });
// send resulting URL to the client
await resultSender.SendSuccessAsync(drawingUrl, stats);
}
/// <summary>
/// Generate Drawing zip with folder structure (or take it from cache).
/// </summary>
public async Task <(FdaStatsDTO stats, string reportUrl)> GenerateDrawingAsync(string projectName, string hash)
{
_logger.LogInformation($"Generating Drawing for hash {hash}");
ProjectStorage storage = await _userResolver.GetProjectStorageAsync(projectName);
Project project = storage.Project;
var ossNames = project.OssNameProvider(hash);
var bucket = await _userResolver.GetBucketAsync();
// check if Drawing file is already generated
if (await bucket.ObjectExistsAsync(ossNames.Drawing))
{
var stats = await bucket.DeserializeAsync <Statistics[]>(ossNames.StatsDrawings);
return(FdaStatsDTO.CreditsOnly(stats), null);
}
// OK, nothing in cache - generate it now
var inputDocUrl = await bucket.CreateSignedUrlAsync(ossNames.GetCurrentModel(storage.IsAssembly));
ProcessingArgs drawingData = await _arranger.ForDrawingAsync(inputDocUrl, storage.Metadata.TLA);
ProcessingResult result = await _fdaClient.GenerateDrawing(drawingData);
if (!result.Success)
{
_logger.LogError($"{result.ErrorMessage} for project {project.Name} and hash {hash}");
throw new FdaProcessingException($"{result.ErrorMessage} for project {project.Name} and hash {hash}", result.ReportUrl);
}
await _arranger.MoveDrawingAsync(project, hash);
await bucket.UploadAsJsonAsync(ossNames.StatsDrawings, result.Stats);
return(FdaStatsDTO.All(result.Stats), result.ReportUrl);
}
public void InvalidInput()
{
// it's expected that statistics is for successful jobs. Missing field(s) should throw an error.
Assert.ThrowsAny <Exception>(() => FdaStatsDTO.All(new List <Statistics>(new [] { new Statistics() })));
}
public void Empty()
{
Assert.Null(FdaStatsDTO.All(new List <Statistics>()));
}
public void Null()
{
Assert.Null(FdaStatsDTO.All(null));
}
public async Task <(FdaStatsDTO stats, int drawingIdx, string reportUrl)> ExportDrawingPdfAsync(string projectName, string hash, string drawingKey)
{
_logger.LogInformation($"Getting drawing pdf for hash {hash}");
ProjectStorage storage = await _userResolver.GetProjectStorageAsync(projectName);
Project project = storage.Project;
var ossNames = project.OssNameProvider(hash);
var ossAttributes = project.OssAttributes;
// get drawing index from drawing specified
var bucket = await _userResolver.GetBucketAsync();
var localAttributes = project.LocalAttributes;
// read cached drawingsList
var drawings = Json.DeserializeFile <List <string> >(localAttributes.DrawingsList);
int index = drawings.IndexOf(drawingKey);
var drawingIdx = index >= 0 ? index : 0;
// check if Drawing viewables file is already generated
try
{
bool generated = false;
ApiResponse <dynamic> ossObjectResponse = await bucket.GetObjectAsync(ossNames.DrawingPdf(drawingIdx));
if (ossObjectResponse != null)
{
await using Stream objectStream = ossObjectResponse.Data;
// zero length means that there is nothing to generate, but processed and do not continue
generated = objectStream.Length > 0;
}
if (generated)
{
var nativeStats = await bucket.DeserializeAsync <List <Statistics> >(ossNames.StatsDrawingPDF(drawingIdx));
return(FdaStatsDTO.CreditsOnly(nativeStats), drawingIdx, null);
}
else
{
return(null, drawingIdx, null);
}
}
catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound)
{
// the file does not exist, so just swallow
}
_logger.LogInformation($"Drawing PDF for hash {hash}: generating");
// OK, nothing in cache - generate it now
var inputDocUrl = await bucket.CreateSignedUrlAsync(ossNames.GetCurrentModel(storage.IsAssembly));
ProcessingArgs drawingData = await _arranger.ForDrawingPdfAsync(inputDocUrl, drawingKey, storage.Metadata.TLA);
ProcessingResult result = await _fdaClient.ExportDrawingAsync(drawingData);
if (!result.Success)
{
_logger.LogError($"{result.ErrorMessage} for project {project.Name} and hash {hash}");
throw new FdaProcessingException($"{result.ErrorMessage} for project {project.Name} and hash {hash}", result.ReportUrl);
}
// move to the right place
await _arranger.MoveDrawingPdfAsync(project, drawingIdx, hash);
// check if Drawing PDF file is generated
try
{
await bucket.CreateSignedUrlAsync(ossNames.DrawingPdf(drawingIdx));
// handle statistics
await bucket.UploadAsJsonAsync(ossNames.StatsDrawingPDF(drawingIdx), result.Stats);
_logger.LogInformation($"Drawing PDF for hash {hash} is generated");
return(FdaStatsDTO.All(result.Stats), drawingIdx, result.ReportUrl);
}
catch (ApiException e) when(e.ErrorCode == StatusCodes.Status404NotFound)
{
// the file does not exist after generating drawing, so just mark with zero length that we already processed it
await bucket.UploadObjectAsync(ossNames.DrawingPdf(drawingIdx), new MemoryStream(0));
_logger.LogError($"Drawing PDF for hash {hash} is NOT generated");
return(null, drawingIdx, result.ReportUrl);
}
}
