public void Test_TAGFileBufferQueueGrouper_AddAndExtractTagFile()
{
TAGFileBufferQueueGrouper grouper = new TAGFileBufferQueueGrouper();
const string tagFileName = "TestTAGFile - TAGFile - Read - Stream.tag";
Guid projectID = Guid.NewGuid();
Guid assetID = Guid.NewGuid();
ITAGFileBufferQueueKey tagKey = new TAGFileBufferQueueKey(tagFileName, projectID, assetID);
grouper.Add(tagKey);
// Test the project is not returned if it is included in the avoid list
var noTagFiles = grouper.Extract(new List <Guid> {
projectID
}, out Guid noProjectUID)?.ToList();
Assert.True(null == noTagFiles, $"Extract from grouper with avoided project {projectID} returned a result for project {noProjectUID}");
// Test the key is present in the extracted list of tag files for the given project
var tagFiles = grouper.Extract(new List <Guid>(), out Guid extractedProjectID)?.ToList();
Assert.True(null != tagFiles, "Returned list of grouped tag files is null");
Assert.True(1 == tagFiles.Count, $"Returned list of grouped tag files does not have a single item (count = {tagFiles.Count}");
Assert.True(extractedProjectID == tagFiles[0].ProjectUID, $"Project UID does not match project UID out parameter from extract call {extractedProjectID} versus {tagFiles[0].ProjectUID}");
Assert.True(tagKey.AssetUID == tagFiles[0].AssetUID, $"Asset UIDs do not match {tagKey.AssetUID} versus {tagFiles[0].AssetUID}");
Assert.True(tagKey.ProjectUID == tagFiles[0].ProjectUID, $"Project UIDs do not match {tagKey.ProjectUID} versus {tagFiles[0].ProjectUID}");
Assert.True(tagKey.FileName == tagFiles[0].FileName, $"File names do not match {tagKey.FileName} versus {tagFiles[0].FileName}");
//Test there are no more TAG files to extract from the grouper
var tagFiles2 = grouper.Extract(null, out Guid _)?.ToList();
Assert.True(null == tagFiles2, "Extract from empty grouper returned a non null result");
}
public void Test_TAGFileBufferQueue_AddingTAGFile()
{
EnsureServer();
var queue = new TAGFileBufferQueue();
Assert.NotNull(queue);
// Load a TAG file and add it to the queue. Verify the TAG file appears in the cache
var tagFileName = "TestTAGFile-TAGFile-Read-Stream.tag";
var projectUid = Guid.NewGuid();
var assetUid = Guid.NewGuid();
byte[] tagContent;
using (var tagFileStream =
new FileStream(Path.Combine("TestData", "TAGFiles", tagFileName),
FileMode.Open, FileAccess.Read))
{
tagContent = new byte[tagFileStream.Length];
tagFileStream.Read(tagContent, 0, (int)tagFileStream.Length);
}
var tagKey = new TAGFileBufferQueueKey(tagFileName, projectUid, assetUid);
var tagItem = new TAGFileBufferQueueItem
{
InsertUTC = DateTime.UtcNow,
ProjectID = projectUid,
AssetID = assetUid,
FileName = tagFileName,
Content = tagContent
};
// Perform the actual add
queue.Add(tagKey, tagItem);
// Read it back from the cache to ensure it was added as expected.
var queueCache = _ignite.GetCache <ITAGFileBufferQueueKey, TAGFileBufferQueueItem>(TRexCaches.TAGFileBufferQueueCacheName());
var tagItem2 = queueCache.Get(tagKey);
Assert.True(tagItem2 != null, "Tag item read back from buffer queue cache was null");
Assert.True(tagItem.Content.Length == tagItem2.Content.Length, "Tag content lengths different");
Assert.True(tagItem.InsertUTC == tagItem2.InsertUTC, "Tag insert UTCs different");
Assert.True(tagItem.AssetID == tagItem2.AssetID, "Tag AssetUIDs different");
Assert.True(tagItem.FileName == tagItem2.FileName, "Tag FileNames different");
Assert.True(tagItem.ProjectID == tagItem2.ProjectID, "Tag ProjectUIDs different");
}
private async Task ProcessTAGFilesFromGrouper()
{
_log.LogInformation("ProcessTAGFilesFromGrouper starting executing");
ITAGFileBufferQueueKey removalKey = new TAGFileBufferQueueKey();
// Cycle looking for new work to do as TAG files arrive until aborted...
do
{
var hadWorkToDo = false;
// Check to see if there is a work package to feed to the processing pipeline
// -> Ask the grouper for a package
var package = _grouper.Extract(_projectsToAvoid, out var projectId)?.ToList();
var packageCount = package?.Count ?? 0;
if (packageCount > 0)
{
_log.LogInformation($"Extracted package from grouper, ProjectUID:{projectId}, with {packageCount} items");
hadWorkToDo = true;
try
{
List <TAGFileBufferQueueItem> tagQueueItems = null;
List <ProcessTAGFileRequestFileItem> fileItems = null;
try
{
tagQueueItems = package?.Select(x =>
{
try
{
return(_queueCache.Get(x));
}
catch (KeyNotFoundException)
{
// Odd, but let's be graceful and attempt to process the remainder in the package
_log.LogError($"Error, KeyNotFoundException exception occurred while attempting to retrieve TAG file for key {x} from the TAG file buffer queue cache");
return(null);
}
catch (Exception e)
{
// More worrying, report and bail on this package
_log.LogError(e, $"Error, exception occurred while attempting to retrieve TAG file for key {x} from the TAG file buffer queue cache - aborting processing this package");
throw;
}
}).ToList();
fileItems = tagQueueItems?
.Where(x => x != null)
.Select(x => new ProcessTAGFileRequestFileItem
{
FileName = x.FileName,
TagFileContent = x.Content,
IsJohnDoe = x.IsJohnDoe,
AssetId = x.AssetID,
SubmissionFlags = x.SubmissionFlags,
OriginSource = x.OriginSource
}).ToList();
}
catch (Exception e)
{
_log.LogError(e, "Error, exception occurred while attempting to retrieve TAG files from the TAG file buffer queue cache");
}
if (tagQueueItems?.Count > 0)
{
// -> Supply the package to the processor
var request = new ProcessTAGFileRequest();
var response = await request.ExecuteAsync(new ProcessTAGFileRequestArgument
{
ProjectID = projectId,
TAGFiles = fileItems
});
removalKey.ProjectUID = projectId;
// -> Remove the set of processed TAG files from the buffer queue cache (depending on processing status?...)
foreach (var tagFileResponse in response.Results)
{
try
{
if (tagFileResponse.Success)
{
//Commented out to keep happy path log less noisy
_log.LogInformation($"Grouper1 TAG file {tagFileResponse.FileName} successfully processed");
}
else
{
_log.LogError($"Grouper1 TAG file failed to process, with exception {tagFileResponse.Exception}, read result = {tagFileResponse.ReadResult}. WARNING: FILE REMOVED FROM QUEUE");
// TODO: Determine what to do in this failure mode: Leave in place? Copy to dead letter queue? Place in S3 bucket pending downstream handling?
}
removalKey.FileName = tagFileResponse.FileName;
removalKey.AssetUID = tagFileResponse.AssetUid;
_log.LogError(!await _queueCache.RemoveAsync(removalKey) ? $"Failed to remove TAG file {removalKey}" : $"Successfully removed TAG file {removalKey}");
}
catch (Exception e)
{
_log.LogError(e, $"Exception occurred while removing TAG file {tagFileResponse.FileName} in project {projectId} from the TAG file buffer queue");
}
}
}
}
finally
{
// Remove the project from the avoid list
_log.LogInformation($"Thread {Thread.CurrentThread.ManagedThreadId}: About to remove project {projectId} from [{(!_projectsToAvoid.Any() ? "Empty" : _projectsToAvoid.Select(x => $"{x}").Aggregate((a, b) => $"{a} + {b}"))}]");
_grouper.RemoveProjectFromAvoidList(_projectsToAvoid, projectId);
}
/// <summary>
/// Receive a TAG file to be processed, validate TAG File Authorization for the file, and add it to the
/// queue to be processed.
/// </summary>
/// <param name="projectId">Project ID to be used as an override to any project ID that may be determined via TAG file authorization</param>
/// <param name="assetId">Asset ID to be used as an override to any Asset ID that may be determined via TAG file authorization</param>
/// <param name="tagFileName">Name of the physical tag file for archiving and logging</param>
/// <param name="tagFileContent">The content of the TAG file to be processed, expressed as a byte array</param>
/// <param name="tccOrgId">Used by TFA service to match VL customer to TCC org when looking for project if multiple projects and/or machine ID not in tag file</param>
/// <param name="treatAsJohnDoe">The TAG file will be processed as if it were a john doe machine is projectId is also specified</param>
/// <param name="tagFileSubmissionFlags">A flag set controlling how certain aspects of managing a submitted TAG file should be managed</param>
/// <param name="originSource">Indictaes the system of origin this file came from</param>
public async Task <SubmitTAGFileResponse> ExecuteAsync(Guid?projectId, Guid?assetId, string tagFileName, byte[] tagFileContent,
string tccOrgId, bool treatAsJohnDoe, TAGFileSubmissionFlags tagFileSubmissionFlags, TAGFileOriginSource originSource)
{
if (OutputInformationalRequestLogging)
{
_log.LogInformation($"#In# SubmitTAGFileResponse. Processing {tagFileName} TAG file into ProjectUID:{projectId}, asset:{assetId}, John Doe?:{treatAsJohnDoe}, submission flags: {tagFileSubmissionFlags}, origin source:{originSource}");
}
var response = new SubmitTAGFileResponse
{
FileName = tagFileName,
Success = false,
Message = "TRex unknown result (SubmitTAGFileResponse.Execute)",
Code = (int)TRexTagFileResultCode.TRexUnknownException,
};
try
{
try
{
// wrap up details into obj
var td = new TagFileDetail
{
assetId = assetId,
projectId = projectId,
tagFileName = tagFileName,
tagFileContent = tagFileContent,
tccOrgId = tccOrgId,
IsJohnDoe = treatAsJohnDoe
};
ContractExecutionResult result;
if (originSource == TAGFileOriginSource.LegacyTAGFileSource)
{
// Validate tag file submission
result = TagfileValidator.PreScanTagFile(td, out var tagFilePreScan);
if (result.Code == (int)TRexTagFileResultCode.Valid)
{
if (_isDeviceGatewayEnabled)
{
SendDeviceStatusToDeviceGateway(td, tagFilePreScan);
}
result = await TagfileValidator.ValidSubmission(td, tagFilePreScan);
}
}
else
{
// For non legacy origins where we have no overt validation rules or need for device status notifications
// note the presented file as valid for processing
result = new ContractExecutionResult((int)TRexTagFileResultCode.Valid, "Success");
}
response.Code = result.Code;
response.Message = result.Message;
if (result.Code == (int)TRexTagFileResultCode.Valid && td.projectId != null) // If OK add to process queue
{
// First archive the tag file
if (_tagFileArchiving && tagFileSubmissionFlags.HasFlag(TAGFileSubmissionFlags.AddToArchive)
// For now, GCS900/Earthworks style TAG files are the only ones archived
&& originSource == TAGFileOriginSource.LegacyTAGFileSource)
{
_log.LogInformation($"#Progress# SubmitTAGFileResponse. Archiving tag file:{tagFileName}, ProjectUID:{td.projectId}");
if (!await TagFileRepository.ArchiveTagfileS3(td))
{
_log.LogError($"SubmitTAGFileResponse. Failed to archive tag file. Returning TRexQueueSubmissionError error. ProjectUID:{td.projectId}, AssetUID:{td.assetId}, Tagfile:{tagFileName}");
throw new ServiceException(HttpStatusCode.InternalServerError, new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, $"SubmitTAGFileResponse. Failed to archive tag file {tagFileName} to S3"));
}
}
// switch from nullable to not nullable
var validProjectId = td.projectId ?? Guid.Empty;
var validAssetId = td.assetId ?? Guid.Empty;
if (OutputInformationalRequestLogging)
{
_log.LogInformation($"#Progress# SubmitTAGFileResponse. Submitting tag file to TagFileBufferQueue. ProjectUID:{validProjectId}, AssetUID:{validAssetId}, Tagfile:{tagFileName}, JohnDoe:{td.IsJohnDoe} ");
}
var tagKey = new TAGFileBufferQueueKey(tagFileName, validProjectId, validAssetId);
var tagItem = new TAGFileBufferQueueItem
{
InsertUTC = DateTime.UtcNow,
ProjectID = validProjectId,
AssetID = validAssetId,
FileName = tagFileName,
Content = tagFileContent,
IsJohnDoe = td.IsJohnDoe,
SubmissionFlags = tagFileSubmissionFlags,
OriginSource = originSource
};
if (_queue == null)
{
throw new ServiceException(HttpStatusCode.InternalServerError, new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, "SubmitTAGFileResponse. Processing queue not available"));
}
if (_queue.Add(tagKey, tagItem)) // Add tag file to queue
{
response.Success = true;
response.Message = "";
response.Code = (int)TRexTagFileResultCode.Valid;
// Commented out top reduce logging
// Log.LogInformation($"Added TAG file {tagKey.FileName} representing asset {tagKey.AssetUID} within project {tagKey.ProjectUID} into the buffer queue");
}
else
{
response.Success = false;
response.Message = "SubmitTAGFileResponse. Failed to submit tag file to processing queue. Request already exists";
response.Code = (int)TRexTagFileResultCode.TRexQueueSubmissionError;
_log.LogWarning(response.Message);
}
}
else
{
response.Success = false;
}
}
catch (Exception e) // catch all exceptions here
{
_log.LogError(e, $"#Exception# SubmitTAGFileResponse. Exception occured processing {tagFileName} Exception:");
throw new ServiceException(HttpStatusCode.InternalServerError, new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, $"SubmitTAGFileResponse. Exception {e.Message}"));
}
}
finally
{
if (OutputInformationalRequestLogging)
{
_log.LogInformation($"#Out# SubmitTAGFileResponse. Processed {tagFileName} Result: {response.Success}, Message:{response.Message} Code:{response.Code}");
}
}
return(response);
}
