public void Test_AggregatedDataIntegratorWorker_ProcessTask_SingleTAGFile_JohnDoe()
{
InjectCoordinateService();
var newMachineId = Guid.NewGuid();
// Convert a TAG file using a TAGFileConverter into a mini-site model
var converter = DITagFileFixture.ReadTAGFile("TestTAGFile.tag", newMachineId, true);
var testTAGFileMachineID = "CB54XW JLM00885";
// Create the site model and machine etc to aggregate the processed TAG file into
var targetSiteModel = BuildModel();
// Create the integrator and add the processed TAG file to its processing list
var integrator = new AggregatedDataIntegrator();
integrator.AddTaskToProcessList(converter.SiteModel, targetSiteModel.ID, converter.Machines,
converter.SiteModelGridAggregator, converter.ProcessedCellPassCount, converter.MachinesTargetValueChangesAggregator);
// Construct an integration worker and ask it to perform the integration
var processedTasks = new List <AggregatedDataIntegratorTask>();
var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, targetSiteModel.ID);
worker.ProcessTask(processedTasks, 1);
processedTasks.Count.Should().Be(1);
targetSiteModel.Grid.CountLeafSubGridsInMemory().Should().Be(12);
targetSiteModel.Machines.Count.Should().Be(1);
targetSiteModel.Machines[0].ID.Should().NotBe(Guid.Empty);
targetSiteModel.Machines[0].InternalSiteModelMachineIndex.Should().Be(0);
targetSiteModel.Machines[0].IsJohnDoeMachine.Should().BeTrue();
targetSiteModel.Machines[0].Name.Should().Be(testTAGFileMachineID);
}
public void Test_AggregatedDataIntegratorWorker_ProcessTask_SingleTAGFileTwice()
{
InjectCoordinateService();
var newMachineId = Guid.NewGuid();
// Convert a TAG file using a TAGFileConverter into a mini-site model
var converter1 = DITagFileFixture.ReadTAGFile("TestTAGFile.tag", newMachineId, false);
var converter2 = DITagFileFixture.ReadTAGFile("TestTAGFile.tag", newMachineId, false);
// Create the site model and machine etc to aggregate the processed TAG file into
var targetSiteModel = BuildModel();
targetSiteModel.Machines.CreateNew("Test Machine", string.Empty, MachineType.Dozer, DeviceTypeEnum.SNM940, false, newMachineId);
// Create the integrator and add the processed TAG file to its processing list
var integrator = new AggregatedDataIntegrator();
integrator.AddTaskToProcessList(converter1.SiteModel, targetSiteModel.ID, converter1.Machines,
converter1.SiteModelGridAggregator, converter1.ProcessedCellPassCount, converter1.MachinesTargetValueChangesAggregator);
integrator.AddTaskToProcessList(converter2.SiteModel, targetSiteModel.ID, converter2.Machines,
converter2.SiteModelGridAggregator, converter2.ProcessedCellPassCount, converter2.MachinesTargetValueChangesAggregator);
// Construct an integration worker and ask it to perform the integration
var processedTasks = new List <AggregatedDataIntegratorTask>();
var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, targetSiteModel.ID);
worker.ProcessTask(processedTasks, 2);
processedTasks.Count.Should().Be(2);
targetSiteModel.Grid.CountLeafSubGridsInMemory().Should().Be(12);
}
public void Test_AggregatedDataIntegrator_Creation()
{
AggregatedDataIntegrator integrator = new AggregatedDataIntegrator();
Assert.True(integrator.CanAcceptMoreAggregatedCellPasses, "CanAcceptMoreAggregatedCellPasses is false");
Assert.Equal(0, integrator.CountOfTasksToProcess);
}
public void Test_AggregatedDataIntegratorWorker_ProcessTask_TAGFileSet_MultipleMachines()
{
// Convert TAG files using TAGFileConverters into mini-site models
using var targetSiteModel = BuildModel();
var integrator = new AggregatedDataIntegrator();
void Convert(string tagFileCollectionFolder, int skipTo, int numToTake)
{
var converters = Directory.GetFiles(Path.Combine("TestData", "TAGFiles", tagFileCollectionFolder), "*.tag")
.OrderBy(x => x).Skip(skipTo).Take(numToTake).Select(x => DITagFileFixture.ReadTAGFileFullPath(x)).ToArray();
converters.Length.Should().Be(numToTake);
var machineGuid = Guid.NewGuid();
var targetMachine = targetSiteModel.Machines.CreateNew($"Test Machine {machineGuid}", $"{machineGuid}", MachineType.Dozer,
DeviceTypeEnum.SNM940, false, machineGuid);
// Add the processed TAG files to the integrator processing list
foreach (var c in converters)
{
using (c)
{
c.Machine.ID = targetMachine.ID;
integrator.AddTaskToProcessList(c.SiteModel, targetSiteModel.ID, c.Machines,
c.SiteModelGridAggregator, c.ProcessedCellPassCount, c.MachinesTargetValueChangesAggregator);
}
}
}
Convert("Dimensions2018-CaseMachine", 0, 10);
Convert("Dimensions2018-CaseMachine", 10, 10);
Convert("Dimensions2018-CaseMachine", 20, 10);
Convert("Dimensions2018-CaseMachine", 30, 10);
// Construct an integration worker and ask it to perform the integration
var processedTasks = new List <AggregatedDataIntegratorTask>();
var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, targetSiteModel.ID)
{
MaxMappedTagFilesToProcessPerAggregationEpoch = integrator.TasksToProcess.Count
};
worker.ProcessTask(processedTasks, 40);
// Pretend all sub grids were saved... (calling CompleteTaskProcessing() will drop the site model)
targetSiteModel.Grid.ScanAllSubGrids(
subGrid =>
{
subGrid.AllChangesMigrated();
return(true);
});
// Check the set of TAG files created the expected number of sub grids and machines
targetSiteModel.Machines.Count.Should().Be(4);
targetSiteModel.ExistenceMap.CountBits().Should().Be(5);
}
public void Test_AggregatedDataIntegrator_IncrementOutstandingCellPasses()
{
AggregatedDataIntegrator integrator = new AggregatedDataIntegrator();
integrator.IncrementOutstandingCellPasses(1000);
integrator.GetStatistics(out int outstandingCellPasses, out long totalCellPassesProcessed, out int pendingFilesToBeProcessed);
Assert.Equal(1000, outstandingCellPasses);
}
[InlineData("Dimensions2018-CaseMachine", 164, 164, 0, 164, 9)] // Take the lot
public void Test_AggregatedDataIntegratorWorker_ProcessTask_TAGFileSet(string tagFileCollectionFolder,
int expectedFileCount, int maxTagFilesPerAggregation, int skipTo, int numToTake, int expectedSubGridCount)
{
Directory.GetFiles(Path.Combine("TestData", "TAGFiles", "Dimensions2018-CaseMachine"), "*.tag").Length.Should().Be(expectedFileCount);
// Log.LogInformation($"Starting processing {numToTake} files from index {skipTo}.");
// Convert TAG files using TAGFileConverters into mini-site models
var converters = Directory.GetFiles(Path.Combine("TestData", "TAGFiles", tagFileCollectionFolder), "*.tag")
.OrderBy(x => x).Skip(skipTo).Take(numToTake).Select(x => DITagFileFixture.ReadTAGFileFullPath(x)).ToArray();
// Log.LogInformation($"Completed constructing converters for {numToTake} files from index {skipTo}.");
converters.Length.Should().Be(numToTake);
// Create the site model and machine etc to aggregate the processed TAG file into
using var targetSiteModel = BuildModel();
var targetMachine = targetSiteModel.Machines.CreateNew("Test Machine", string.Empty, MachineType.Dozer,
DeviceTypeEnum.SNM940, false, Guid.NewGuid());
// Create the integrator and add the processed TAG file to its processing list
var integrator = new AggregatedDataIntegrator();
foreach (var c in converters)
{
using (c)
{
c.Machine.ID = targetMachine.ID;
integrator.AddTaskToProcessList(c.SiteModel, targetSiteModel.ID, c.Machines,
c.SiteModelGridAggregator, c.ProcessedCellPassCount, c.MachinesTargetValueChangesAggregator);
}
}
// Construct an integration worker and ask it to perform the integration
var processedTasks = new List <AggregatedDataIntegratorTask>();
var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, targetSiteModel.ID)
{
MaxMappedTagFilesToProcessPerAggregationEpoch = maxTagFilesPerAggregation
};
// Log.LogInformation("Calling ProcessTask");
worker.ProcessTask(processedTasks, converters.Length);
// Log.LogInformation("Calling CompleteTaskProcessing");
processedTasks.Count.Should().Be(numToTake);
// Check the set of TAG files created the expected number of sub grids
targetSiteModel.Grid.CountLeafSubGridsInMemory().Should().Be(expectedSubGridCount);
// Log.LogInformation($"Completed processing {numToTake} files.");
}
public void Test_AggregatedDataIntegrator_AddTaskToProcessList()
{
var integrator = new AggregatedDataIntegrator();
SiteModel siteModel = new SiteModel(/*"TestName", "TestDesc", */ Guid.NewGuid(), StorageMutability.Immutable, 1.0);
IMachinesList machines = new MachinesList();
machines.Add(new Machine("TestName", "TestHardwareID", 0, 0, Guid.NewGuid(), 0, false));
ServerSubGridTree tree = new ServerSubGridTree(siteModel.ID, StorageMutability.Mutable);
MachinesProductionEventLists events = new MachinesProductionEventLists(siteModel, 1);
integrator.AddTaskToProcessList(siteModel, siteModel.ID, machines, tree, 0, events);
Assert.Equal(1, integrator.CountOfTasksToProcess);
Assert.True(integrator.CanAcceptMoreAggregatedCellPasses, "CanAcceptMoreAggregatedCellPasses is false");
}
/// <summary>
/// Takes a list of TAG files and constructs an ephemeral site model that may be queried
/// </summary>
public static ISiteModel BuildModel(IEnumerable <string> tagFiles, out List <AggregatedDataIntegratorTask> ProcessedTasks,
bool callTaskProcessingComplete = true,
bool convertToImmutableRepresentation = true,
bool treatAsJohnDoeMachines = false)
{
ProcessedTasks = null;
var _tagFiles = tagFiles.ToList();
// Create the site model and machine etc to aggregate the processed TAG file into
var targetSiteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(NewSiteModelGuid, true);
var preTargetSiteModelId = targetSiteModel.ID;
// Switch to mutable storage representation to allow creation of content in the site model
targetSiteModel.SetStorageRepresentationToSupply(StorageMutability.Mutable);
targetSiteModel.ID.Should().Be(preTargetSiteModelId);
targetSiteModel.PrimaryStorageProxy.Mutability.Should().Be(StorageMutability.Mutable);
targetSiteModel.PrimaryStorageProxy.ImmutableProxy.Should().NotBeNull();
IMachine targetMachine = null;
// Create the integrator and add the processed TAG file to its processing list
var integrator = new AggregatedDataIntegrator();
var pageNumber = 0;
var pageSize = 25;
var taskCount = 0;
TAGFileConverter[] converters = null;
do
{
// Convert TAG files using TAGFileConverters into mini-site models
converters = _tagFiles.Skip(pageNumber * pageSize).Take(pageSize).Select(x => ReadTAGFileFullPath(x, treatAsJohnDoeMachines)).ToArray();
var currentMachineName = string.Empty;
for (var i = 0; i < converters.Length; i++)
{
//One converter per tag file. See if machine changed if using real tag files.
var parts = _tagFiles[i].Split("--");
var tagMachineName = parts.Length == 3 ? parts[1] : "Test Machine";
if (!string.Equals(currentMachineName, tagMachineName, StringComparison.OrdinalIgnoreCase))
{
currentMachineName = tagMachineName;
targetMachine = targetSiteModel.Machines.CreateNew(currentMachineName, "", MachineType.Dozer, DeviceTypeEnum.SNM940, treatAsJohnDoeMachines, Guid.NewGuid());
}
converters[i].Machine.ID = targetMachine.ID;
integrator.AddTaskToProcessList(converters[i].SiteModel, targetSiteModel.ID, converters[i].Machines,
converters[i].SiteModelGridAggregator, converters[i].ProcessedCellPassCount, converters[i].MachinesTargetValueChangesAggregator);
if ((i + 1) % 10 == 0 || i == converters.Length - 1)
{
// Construct an integration worker and ask it to perform the integration
ProcessedTasks = new List <AggregatedDataIntegratorTask>();
var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, targetSiteModel.ID)
{
MaxMappedTagFilesToProcessPerAggregationEpoch = _tagFiles.Count
};
worker.ProcessTask(ProcessedTasks, _tagFiles.Count);
taskCount += ProcessedTasks.Count;
}
}
pageNumber++;
converters.ForEach(x => x.Dispose());
} while (converters.Length > 0);
if (callTaskProcessingComplete)
{
new AggregatedDataIntegratorWorker(null, targetSiteModel.ID).CompleteTaskProcessing();
}
// Construct an integration worker and ask it to perform the integration
taskCount.Should().Be(_tagFiles.Count);
// Reacquire the target site model to ensure any notification based changes to the site model are observed
targetSiteModel.SiteModelExtent.IsValidPlanExtent.Should().BeTrue();
targetSiteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(preTargetSiteModelId, false);
targetSiteModel.SiteModelExtent.IsValidPlanExtent.Should().BeTrue();
// The ISiteModels instance will supply the immutable representation of the site model, ensure it is set to mutable
targetSiteModel.SetStorageRepresentationToSupply(StorageMutability.Mutable);
targetSiteModel.Should().NotBe(null);
targetSiteModel.ID.Should().Be(preTargetSiteModelId);
targetSiteModel.PrimaryStorageProxy.Mutability.Should().Be(StorageMutability.Mutable);
targetSiteModel.PrimaryStorageProxy.ImmutableProxy.Should().NotBeNull();
targetSiteModel.SiteModelExtent.IsValidPlanExtent.Should().BeTrue();
// Modify the site model to switch from the mutable to immutable cell pass representation for read requests
if (convertToImmutableRepresentation)
{
ConvertSiteModelToImmutable(targetSiteModel);
}
return(targetSiteModel);
}
/*
public static ProcessTAGFileResponse Execute_Legacy_IndividualModelsWithWhenAny(Guid ProjectID, Guid AssetID,
IEnumerable<ProcessTAGFileRequestFileItem> TAGFiles)
{
var _TAGFiles = TAGFiles.ToArray(); // Enumerate collection just once
Log.LogInformation(
$"ProcessTAGFileResponse.Execute. Processing {_TAGFiles.Count()} TAG files into project {ProjectID}, asset {AssetID}");
var response = new ProcessTAGFileResponse();
int batchCount = 0;
// Create the machinery responsible for tracking tasks and integrating them into the database
var integrator = new AggregatedDataIntegrator();
var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, ProjectID);
var ProcessedTasks = new List<AggregatedDataIntegratorTask>();
// Create the site model and machine etc to aggregate the processed TAG file into
// Note: This creates these elements within the project itself, not just class instances...
// SiteModel siteModel = SiteModels.SiteModels.Instance(StorageMutability.Mutable).GetSiteModel(ProjectUID, true);
// Machine machine = new Machine(null, "TestName", "TestHardwareID", 0, 0, Guid.NewGuid(), 0, false);
// Create a list of tasks to represent conversion of each of the TAGFiles into a min-model
Log.LogInformation($"#Progress# Initiating task based conversion of TAG files into project {ProjectID}");
try
{
var tagFileConversions = _TAGFiles.Select(x => Task.Run(() =>
{
Log.LogInformation($"#Progress# Processing TAG file {x.FileName} into project {ProjectID}");
var converter = new TAGFileConverter();
using (var fs = new MemoryStream(x.TagFileContent))
{
converter.Execute(fs);
Log.LogInformation(
$"#Progress# TAG file {x.FileName} generated {converter.ProcessedCellPassCount} cell passes from {converter.ProcessedEpochCount} epochs");
}
return (x, converter);
})).ToList();
// Pick off the tasks as they complete and pass the results into the integrator processor
while (tagFileConversions.Count > 0)
{
async Task<(ProcessTAGFileRequestFileItem, TAGFileConverter)> nextTaskCompleted()
{
var theTask = await Task.WhenAny(tagFileConversions);
tagFileConversions.Remove(theTask);
return await theTask;
}
var convertedTask = nextTaskCompleted();
var converter = convertedTask.Result.Item2;
var TAGFile = convertedTask.Result.Item1;
try
{
converter.SiteModel.ID = ProjectID;
converter.Machine.ID = AssetID;
converter.Machine.IsJohnDoeMachine = TAGFile.IsJohnDoe;
integrator.AddTaskToProcessList(converter.SiteModel, ProjectID,
converter.Machine, AssetID,
converter.SiteModelGridAggregator,
converter.ProcessedCellPassCount,
converter.MachineTargetValueChangesAggregator);
if (++batchCount >= batchSize)
{
worker.ProcessTask(ProcessedTasks);
batchCount = 0;
}
response.Results.Add(new ProcessTAGFileResponseItem
{
FileName = TAGFile.FileName, Success = converter.ReadResult == TAGReadResult.NoError,
ReadResult = converter.ReadResult
});
}
catch (Exception E)
{
response.Results.Add(new ProcessTAGFileResponseItem
{
FileName = TAGFile.FileName, Success = false, Exception = E.ToString()
});
}
}
if (batchCount > 0)
{
worker.ProcessTask(ProcessedTasks);
}
}
finally
{
worker.CompleteTaskProcessing();
}
Log.LogInformation($"#Progress# Completed task based conversion of TAG files into project {ProjectID}");
return response;
}
*/
public static ProcessTAGFileResponse Execute(Guid projectId, IEnumerable<ProcessTAGFileRequestFileItem> tagFiles)
{
var tagFilesArray = tagFiles.ToArray(); // Enumerate collection just once
if (tagFilesArray.Length == 0)
{
// Send back a response with an empty list
return new ProcessTAGFileResponse
{
Results = new List<IProcessTAGFileResponseItem>()
};
}
_log.LogInformation($"ProcessTAGFileResponse.Execute. Processing {tagFilesArray.Length} TAG files into project {projectId}");
var response = new ProcessTAGFileResponse();
// Create the machinery responsible for tracking tasks and integrating them into the database
var integrator = new AggregatedDataIntegrator();
var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, projectId);
var processedTasks = new List<AggregatedDataIntegratorTask>();
// Create the site model and machine etc to aggregate the processed TAG file into
// Note: This creates these elements within the project itself, not just class instances...
// SiteModel siteModel = SiteModels.SiteModels.Instance(StorageMutability.Mutable).GetSiteModel(ProjectUID, true);
// Machine machine = new Machine(null, "TestName", "TestHardwareID", 0, 0, Guid.NewGuid(), 0, false);
// Assertion: All TAG files relate to the same machine in the same project
// Progressively process each TAG file into the same intermediary site model before integrating that intermediary model
// into the primary persistent model.
// TODO: Failure of a single TAG file may result in contamination of the results of previous processed TAG files requiring exclusion of the TAG file in question and reprocessing of the list
_log.LogInformation($"#Progress# Initiating task based conversion of TAG files into project {projectId}");
try
{
using (var commonConverter = new TAGFileConverter(projectId))
{
foreach (var tagFile in tagFilesArray)
{
using var fs = new MemoryStream(tagFile.TagFileContent);
try
{
_log.LogInformation($"About to read file {tagFile.FileName} with origin source {tagFile.OriginSource}");
var readResult = tagFile.OriginSource switch
{
TAGFileOriginSource.VolvoMachineAssistEarthworksCSV => commonConverter.ExecuteVolvoEarthworksCSVFile(tagFile.FileName, fs, tagFile.AssetId, tagFile.IsJohnDoe),
TAGFileOriginSource.VolvoMachineAssistCompactionCSV => false,
TAGFileOriginSource.LegacyTAGFileSource => commonConverter.ExecuteLegacyTAGFile(tagFile.FileName, fs, tagFile.AssetId, tagFile.IsJohnDoe),
_ => throw new NotImplementedException("Unsupported commonConverter origin source"),
};
response.Results.Add(new ProcessTAGFileResponseItem
{
FileName = tagFile.FileName,
AssetUid = tagFile.AssetId,
ReadResult = commonConverter.ReadResult,
Success = commonConverter.ReadResult == TAGReadResult.NoError && readResult == true,
SubmissionFlags = tagFile.SubmissionFlags,
OriginSource = tagFile.OriginSource
});
_log.LogInformation(
$"#Progress# [CommonConverter] TAG file {tagFile.FileName} generated {commonConverter.ProcessedCellPassCount} cell passes from {commonConverter.ProcessedEpochCount} epochs for asset {tagFile.AssetId} machinetype {commonConverter.Machine?.MachineType.ToString() ?? "null"} with result {commonConverter.ReadResult}");
}
catch (Exception e)
{
_log.LogError(e, $"Processing of TAG file {tagFile.FileName} failed with exception {e.Message}");
response.Results.Add(new ProcessTAGFileResponseItem
{
FileName = tagFile.FileName,
Success = false,
Exception = e.Message,
ReadResult = commonConverter.ReadResult,
SubmissionFlags = tagFile.SubmissionFlags,
OriginSource = tagFile.OriginSource
});
}
}
commonConverter.SiteModel.ID = projectId;
integrator.AddTaskToProcessList(commonConverter.SiteModel, projectId,
commonConverter.Machines,
commonConverter.SiteModelGridAggregator,
commonConverter.ProcessedCellPassCount,
commonConverter.MachinesTargetValueChangesAggregator);
}
worker.ProcessTask(processedTasks, tagFilesArray.Length);
}
finally
{
worker.CompleteTaskProcessing();
}
_log.LogInformation($"#Progress# Completed task based conversion of TAG files into project {projectId}");
return response;
}
public void Test_AggregatedDataIntegratorWorker_AggregatedDataIntegratorWorkerTest()
{
var integrator = new AggregatedDataIntegrator();
Assert.NotNull(integrator);
}
