public static void ProcessTAGFiles(Guid projectID, string[] files)
{
// Machine machine = new Machine(null, "TestName", "TestHardwareID", 0, 0, Guid.NewGuid(), 0, false);
Guid machineID = AssetOverride == Guid.Empty ? Guid.NewGuid() : AssetOverride;
processTAGFileRequest = processTAGFileRequest ?? new ProcessTAGFileRequest();
ProcessTAGFileRequestArgument arg = new ProcessTAGFileRequestArgument
{
ProjectID = projectID,
AssetUID = machineID,
TAGFiles = new List <ProcessTAGFileRequestFileItem>()
};
foreach (string file in files)
{
using (FileStream fs = new FileStream(file, FileMode.Open, FileAccess.Read))
{
byte[] bytes = new byte[fs.Length];
fs.Read(bytes, 0, bytes.Length);
arg.TAGFiles.Add(new ProcessTAGFileRequestFileItem {
FileName = Path.GetFileName(file), TagFileContent = bytes, IsJohnDoe = false
});
}
}
processTAGFileRequest.Execute(arg);
}
public async Task ProcessSingleTAGFile_FailWithInvalidTAGFile()
{
InjectTAGFileProcessorMock();
var req = new ProcessTAGFileRequest();
var arg = new ProcessTAGFileRequestArgument
{
ProjectID = Guid.NewGuid(),
TAGFiles = new List <ProcessTAGFileRequestFileItem>
{
new ProcessTAGFileRequestFileItem
{
IsJohnDoe = false,
FileName = "ATAGFileName",
TagFileContent = new byte[] { 1, 2, 3, 4, 5 },
SubmissionFlags = TAGFileSubmissionFlags.AddToArchive
}
}
};
var response = await req.ExecuteAsync(arg);
response.Results.Count.Should().Be(1);
response.Results[0].Success.Should().BeFalse();
response.Results[0].Exception.Should().BeNull();
}
public async Task ProcessSingleTAGFile_Success()
{
InjectTAGFileProcessorMock();
var fileName = Path.Combine("TestData", "TAGFiles", "TestTAGFile-TAGFile-Read-Stream.tag");
var req = new ProcessTAGFileRequest();
var arg = new ProcessTAGFileRequestArgument
{
ProjectID = Guid.NewGuid(),
TAGFiles = new List <ProcessTAGFileRequestFileItem>
{
new ProcessTAGFileRequestFileItem
{
IsJohnDoe = false,
FileName = fileName,
TagFileContent = File.ReadAllBytes(fileName),
SubmissionFlags = TAGFileSubmissionFlags.AddToArchive
}
}
};
var response = await req.ExecuteAsync(arg);
response.Results.Count.Should().Be(1);
response.Results[0].Success.Should().BeTrue();
response.Results[0].Exception.Should().BeNull();
}
private async Task ProcessTAGFilesFromGrouper()
{
_log.LogInformation("ProcessTAGFilesFromGrouper starting executing");
ITAGFileBufferQueueKey removalKey = new TAGFileBufferQueueKey();
// Cycle looking for new work to do as TAG files arrive until aborted...
do
{
var hadWorkToDo = false;
// Check to see if there is a work package to feed to the processing pipeline
// -> Ask the grouper for a package
var package = _grouper.Extract(_projectsToAvoid, out var projectId)?.ToList();
var packageCount = package?.Count ?? 0;
if (packageCount > 0)
{
_log.LogInformation($"Extracted package from grouper, ProjectUID:{projectId}, with {packageCount} items");
hadWorkToDo = true;
try
{
List <TAGFileBufferQueueItem> tagQueueItems = null;
List <ProcessTAGFileRequestFileItem> fileItems = null;
try
{
tagQueueItems = package?.Select(x =>
{
try
{
return(_queueCache.Get(x));
}
catch (KeyNotFoundException)
{
// Odd, but let's be graceful and attempt to process the remainder in the package
_log.LogError($"Error, KeyNotFoundException exception occurred while attempting to retrieve TAG file for key {x} from the TAG file buffer queue cache");
return(null);
}
catch (Exception e)
{
// More worrying, report and bail on this package
_log.LogError(e, $"Error, exception occurred while attempting to retrieve TAG file for key {x} from the TAG file buffer queue cache - aborting processing this package");
throw;
}
}).ToList();
fileItems = tagQueueItems?
.Where(x => x != null)
.Select(x => new ProcessTAGFileRequestFileItem
{
FileName = x.FileName,
TagFileContent = x.Content,
IsJohnDoe = x.IsJohnDoe,
AssetId = x.AssetID,
SubmissionFlags = x.SubmissionFlags,
OriginSource = x.OriginSource
}).ToList();
}
catch (Exception e)
{
_log.LogError(e, "Error, exception occurred while attempting to retrieve TAG files from the TAG file buffer queue cache");
}
if (tagQueueItems?.Count > 0)
{
// -> Supply the package to the processor
var request = new ProcessTAGFileRequest();
var response = await request.ExecuteAsync(new ProcessTAGFileRequestArgument
{
ProjectID = projectId,
TAGFiles = fileItems
});
removalKey.ProjectUID = projectId;
// -> Remove the set of processed TAG files from the buffer queue cache (depending on processing status?...)
foreach (var tagFileResponse in response.Results)
{
try
{
if (tagFileResponse.Success)
{
//Commented out to keep happy path log less noisy
_log.LogInformation($"Grouper1 TAG file {tagFileResponse.FileName} successfully processed");
}
else
{
_log.LogError($"Grouper1 TAG file failed to process, with exception {tagFileResponse.Exception}, read result = {tagFileResponse.ReadResult}. WARNING: FILE REMOVED FROM QUEUE");
// TODO: Determine what to do in this failure mode: Leave in place? Copy to dead letter queue? Place in S3 bucket pending downstream handling?
}
removalKey.FileName = tagFileResponse.FileName;
removalKey.AssetUID = tagFileResponse.AssetUid;
_log.LogError(!await _queueCache.RemoveAsync(removalKey) ? $"Failed to remove TAG file {removalKey}" : $"Successfully removed TAG file {removalKey}");
}
catch (Exception e)
{
_log.LogError(e, $"Exception occurred while removing TAG file {tagFileResponse.FileName} in project {projectId} from the TAG file buffer queue");
}
}
}
}
finally
{
// Remove the project from the avoid list
_log.LogInformation($"Thread {Thread.CurrentThread.ManagedThreadId}: About to remove project {projectId} from [{(!_projectsToAvoid.Any() ? "Empty" : _projectsToAvoid.Select(x => $"{x}").Aggregate((a, b) => $"{a} + {b}"))}]");
_grouper.RemoveProjectFromAvoidList(_projectsToAvoid, projectId);
}
public void Creation()
{
var req = new ProcessTAGFileRequest();
req.Should().NotBeNull();
}
