public void Test_AggregatedDataIntegratorWorker_ProcessTask_SingleTAGFileTwice() { InjectCoordinateService(); var newMachineId = Guid.NewGuid(); // Convert a TAG file using a TAGFileConverter into a mini-site model var converter1 = DITagFileFixture.ReadTAGFile("TestTAGFile.tag", newMachineId, false); var converter2 = DITagFileFixture.ReadTAGFile("TestTAGFile.tag", newMachineId, false); // Create the site model and machine etc to aggregate the processed TAG file into var targetSiteModel = BuildModel(); targetSiteModel.Machines.CreateNew("Test Machine", string.Empty, MachineType.Dozer, DeviceTypeEnum.SNM940, false, newMachineId); // Create the integrator and add the processed TAG file to its processing list var integrator = new AggregatedDataIntegrator(); integrator.AddTaskToProcessList(converter1.SiteModel, targetSiteModel.ID, converter1.Machines, converter1.SiteModelGridAggregator, converter1.ProcessedCellPassCount, converter1.MachinesTargetValueChangesAggregator); integrator.AddTaskToProcessList(converter2.SiteModel, targetSiteModel.ID, converter2.Machines, converter2.SiteModelGridAggregator, converter2.ProcessedCellPassCount, converter2.MachinesTargetValueChangesAggregator); // Construct an integration worker and ask it to perform the integration var processedTasks = new List <AggregatedDataIntegratorTask>(); var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, targetSiteModel.ID); worker.ProcessTask(processedTasks, 2); processedTasks.Count.Should().Be(2); targetSiteModel.Grid.CountLeafSubGridsInMemory().Should().Be(12); }
public void Test_AggregatedDataIntegratorWorker_ProcessTask_SingleTAGFile_JohnDoe() { InjectCoordinateService(); var newMachineId = Guid.NewGuid(); // Convert a TAG file using a TAGFileConverter into a mini-site model var converter = DITagFileFixture.ReadTAGFile("TestTAGFile.tag", newMachineId, true); var testTAGFileMachineID = "CB54XW JLM00885"; // Create the site model and machine etc to aggregate the processed TAG file into var targetSiteModel = BuildModel(); // Create the integrator and add the processed TAG file to its processing list var integrator = new AggregatedDataIntegrator(); integrator.AddTaskToProcessList(converter.SiteModel, targetSiteModel.ID, converter.Machines, converter.SiteModelGridAggregator, converter.ProcessedCellPassCount, converter.MachinesTargetValueChangesAggregator); // Construct an integration worker and ask it to perform the integration var processedTasks = new List <AggregatedDataIntegratorTask>(); var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, targetSiteModel.ID); worker.ProcessTask(processedTasks, 1); processedTasks.Count.Should().Be(1); targetSiteModel.Grid.CountLeafSubGridsInMemory().Should().Be(12); targetSiteModel.Machines.Count.Should().Be(1); targetSiteModel.Machines[0].ID.Should().NotBe(Guid.Empty); targetSiteModel.Machines[0].InternalSiteModelMachineIndex.Should().Be(0); targetSiteModel.Machines[0].IsJohnDoeMachine.Should().BeTrue(); targetSiteModel.Machines[0].Name.Should().Be(testTAGFileMachineID); }
public void Test_AggregatedDataIntegratorWorker_ProcessTask_TAGFileSet_MultipleMachines() { // Convert TAG files using TAGFileConverters into mini-site models using var targetSiteModel = BuildModel(); var integrator = new AggregatedDataIntegrator(); void Convert(string tagFileCollectionFolder, int skipTo, int numToTake) { var converters = Directory.GetFiles(Path.Combine("TestData", "TAGFiles", tagFileCollectionFolder), "*.tag") .OrderBy(x => x).Skip(skipTo).Take(numToTake).Select(x => DITagFileFixture.ReadTAGFileFullPath(x)).ToArray(); converters.Length.Should().Be(numToTake); var machineGuid = Guid.NewGuid(); var targetMachine = targetSiteModel.Machines.CreateNew($"Test Machine {machineGuid}", $"{machineGuid}", MachineType.Dozer, DeviceTypeEnum.SNM940, false, machineGuid); // Add the processed TAG files to the integrator processing list foreach (var c in converters) { using (c) { c.Machine.ID = targetMachine.ID; integrator.AddTaskToProcessList(c.SiteModel, targetSiteModel.ID, c.Machines, c.SiteModelGridAggregator, c.ProcessedCellPassCount, c.MachinesTargetValueChangesAggregator); } } } Convert("Dimensions2018-CaseMachine", 0, 10); Convert("Dimensions2018-CaseMachine", 10, 10); Convert("Dimensions2018-CaseMachine", 20, 10); Convert("Dimensions2018-CaseMachine", 30, 10); // Construct an integration worker and ask it to perform the integration var processedTasks = new List <AggregatedDataIntegratorTask>(); var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, targetSiteModel.ID) { MaxMappedTagFilesToProcessPerAggregationEpoch = integrator.TasksToProcess.Count }; worker.ProcessTask(processedTasks, 40); // Pretend all sub grids were saved... (calling CompleteTaskProcessing() will drop the site model) targetSiteModel.Grid.ScanAllSubGrids( subGrid => { subGrid.AllChangesMigrated(); return(true); }); // Check the set of TAG files created the expected number of sub grids and machines targetSiteModel.Machines.Count.Should().Be(4); targetSiteModel.ExistenceMap.CountBits().Should().Be(5); }
[InlineData("Dimensions2018-CaseMachine", 164, 164, 0, 164, 9)] // Take the lot public void Test_AggregatedDataIntegratorWorker_ProcessTask_TAGFileSet(string tagFileCollectionFolder, int expectedFileCount, int maxTagFilesPerAggregation, int skipTo, int numToTake, int expectedSubGridCount) { Directory.GetFiles(Path.Combine("TestData", "TAGFiles", "Dimensions2018-CaseMachine"), "*.tag").Length.Should().Be(expectedFileCount); // Log.LogInformation($"Starting processing {numToTake} files from index {skipTo}."); // Convert TAG files using TAGFileConverters into mini-site models var converters = Directory.GetFiles(Path.Combine("TestData", "TAGFiles", tagFileCollectionFolder), "*.tag") .OrderBy(x => x).Skip(skipTo).Take(numToTake).Select(x => DITagFileFixture.ReadTAGFileFullPath(x)).ToArray(); // Log.LogInformation($"Completed constructing converters for {numToTake} files from index {skipTo}."); converters.Length.Should().Be(numToTake); // Create the site model and machine etc to aggregate the processed TAG file into using var targetSiteModel = BuildModel(); var targetMachine = targetSiteModel.Machines.CreateNew("Test Machine", string.Empty, MachineType.Dozer, DeviceTypeEnum.SNM940, false, Guid.NewGuid()); // Create the integrator and add the processed TAG file to its processing list var integrator = new AggregatedDataIntegrator(); foreach (var c in converters) { using (c) { c.Machine.ID = targetMachine.ID; integrator.AddTaskToProcessList(c.SiteModel, targetSiteModel.ID, c.Machines, c.SiteModelGridAggregator, c.ProcessedCellPassCount, c.MachinesTargetValueChangesAggregator); } } // Construct an integration worker and ask it to perform the integration var processedTasks = new List <AggregatedDataIntegratorTask>(); var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, targetSiteModel.ID) { MaxMappedTagFilesToProcessPerAggregationEpoch = maxTagFilesPerAggregation }; // Log.LogInformation("Calling ProcessTask"); worker.ProcessTask(processedTasks, converters.Length); // Log.LogInformation("Calling CompleteTaskProcessing"); processedTasks.Count.Should().Be(numToTake); // Check the set of TAG files created the expected number of sub grids targetSiteModel.Grid.CountLeafSubGridsInMemory().Should().Be(expectedSubGridCount); // Log.LogInformation($"Completed processing {numToTake} files."); }
public void Test_AggregatedDataIntegrator_AddTaskToProcessList() { var integrator = new AggregatedDataIntegrator(); SiteModel siteModel = new SiteModel(/*"TestName", "TestDesc", */ Guid.NewGuid(), StorageMutability.Immutable, 1.0); IMachinesList machines = new MachinesList(); machines.Add(new Machine("TestName", "TestHardwareID", 0, 0, Guid.NewGuid(), 0, false)); ServerSubGridTree tree = new ServerSubGridTree(siteModel.ID, StorageMutability.Mutable); MachinesProductionEventLists events = new MachinesProductionEventLists(siteModel, 1); integrator.AddTaskToProcessList(siteModel, siteModel.ID, machines, tree, 0, events); Assert.Equal(1, integrator.CountOfTasksToProcess); Assert.True(integrator.CanAcceptMoreAggregatedCellPasses, "CanAcceptMoreAggregatedCellPasses is false"); }
/// <summary> /// Takes a list of TAG files and constructs an ephemeral site model that may be queried /// </summary> public static ISiteModel BuildModel(IEnumerable <string> tagFiles, out List <AggregatedDataIntegratorTask> ProcessedTasks, bool callTaskProcessingComplete = true, bool convertToImmutableRepresentation = true, bool treatAsJohnDoeMachines = false) { ProcessedTasks = null; var _tagFiles = tagFiles.ToList(); // Create the site model and machine etc to aggregate the processed TAG file into var targetSiteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(NewSiteModelGuid, true); var preTargetSiteModelId = targetSiteModel.ID; // Switch to mutable storage representation to allow creation of content in the site model targetSiteModel.SetStorageRepresentationToSupply(StorageMutability.Mutable); targetSiteModel.ID.Should().Be(preTargetSiteModelId); targetSiteModel.PrimaryStorageProxy.Mutability.Should().Be(StorageMutability.Mutable); targetSiteModel.PrimaryStorageProxy.ImmutableProxy.Should().NotBeNull(); IMachine targetMachine = null; // Create the integrator and add the processed TAG file to its processing list var integrator = new AggregatedDataIntegrator(); var pageNumber = 0; var pageSize = 25; var taskCount = 0; TAGFileConverter[] converters = null; do { // Convert TAG files using TAGFileConverters into mini-site models converters = _tagFiles.Skip(pageNumber * pageSize).Take(pageSize).Select(x => ReadTAGFileFullPath(x, treatAsJohnDoeMachines)).ToArray(); var currentMachineName = string.Empty; for (var i = 0; i < converters.Length; i++) { //One converter per tag file. See if machine changed if using real tag files. var parts = _tagFiles[i].Split("--"); var tagMachineName = parts.Length == 3 ? parts[1] : "Test Machine"; if (!string.Equals(currentMachineName, tagMachineName, StringComparison.OrdinalIgnoreCase)) { currentMachineName = tagMachineName; targetMachine = targetSiteModel.Machines.CreateNew(currentMachineName, "", MachineType.Dozer, DeviceTypeEnum.SNM940, treatAsJohnDoeMachines, Guid.NewGuid()); } converters[i].Machine.ID = targetMachine.ID; integrator.AddTaskToProcessList(converters[i].SiteModel, targetSiteModel.ID, converters[i].Machines, converters[i].SiteModelGridAggregator, converters[i].ProcessedCellPassCount, converters[i].MachinesTargetValueChangesAggregator); if ((i + 1) % 10 == 0 || i == converters.Length - 1) { // Construct an integration worker and ask it to perform the integration ProcessedTasks = new List <AggregatedDataIntegratorTask>(); var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, targetSiteModel.ID) { MaxMappedTagFilesToProcessPerAggregationEpoch = _tagFiles.Count }; worker.ProcessTask(ProcessedTasks, _tagFiles.Count); taskCount += ProcessedTasks.Count; } } pageNumber++; converters.ForEach(x => x.Dispose()); } while (converters.Length > 0); if (callTaskProcessingComplete) { new AggregatedDataIntegratorWorker(null, targetSiteModel.ID).CompleteTaskProcessing(); } // Construct an integration worker and ask it to perform the integration taskCount.Should().Be(_tagFiles.Count); // Reacquire the target site model to ensure any notification based changes to the site model are observed targetSiteModel.SiteModelExtent.IsValidPlanExtent.Should().BeTrue(); targetSiteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(preTargetSiteModelId, false); targetSiteModel.SiteModelExtent.IsValidPlanExtent.Should().BeTrue(); // The ISiteModels instance will supply the immutable representation of the site model, ensure it is set to mutable targetSiteModel.SetStorageRepresentationToSupply(StorageMutability.Mutable); targetSiteModel.Should().NotBe(null); targetSiteModel.ID.Should().Be(preTargetSiteModelId); targetSiteModel.PrimaryStorageProxy.Mutability.Should().Be(StorageMutability.Mutable); targetSiteModel.PrimaryStorageProxy.ImmutableProxy.Should().NotBeNull(); targetSiteModel.SiteModelExtent.IsValidPlanExtent.Should().BeTrue(); // Modify the site model to switch from the mutable to immutable cell pass representation for read requests if (convertToImmutableRepresentation) { ConvertSiteModelToImmutable(targetSiteModel); } return(targetSiteModel); }
/* public static ProcessTAGFileResponse Execute_Legacy_IndividualModelsWithWhenAny(Guid ProjectID, Guid AssetID, IEnumerable<ProcessTAGFileRequestFileItem> TAGFiles) { var _TAGFiles = TAGFiles.ToArray(); // Enumerate collection just once Log.LogInformation( $"ProcessTAGFileResponse.Execute. Processing {_TAGFiles.Count()} TAG files into project {ProjectID}, asset {AssetID}"); var response = new ProcessTAGFileResponse(); int batchCount = 0; // Create the machinery responsible for tracking tasks and integrating them into the database var integrator = new AggregatedDataIntegrator(); var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, ProjectID); var ProcessedTasks = new List<AggregatedDataIntegratorTask>(); // Create the site model and machine etc to aggregate the processed TAG file into // Note: This creates these elements within the project itself, not just class instances... // SiteModel siteModel = SiteModels.SiteModels.Instance(StorageMutability.Mutable).GetSiteModel(ProjectUID, true); // Machine machine = new Machine(null, "TestName", "TestHardwareID", 0, 0, Guid.NewGuid(), 0, false); // Create a list of tasks to represent conversion of each of the TAGFiles into a min-model Log.LogInformation($"#Progress# Initiating task based conversion of TAG files into project {ProjectID}"); try { var tagFileConversions = _TAGFiles.Select(x => Task.Run(() => { Log.LogInformation($"#Progress# Processing TAG file {x.FileName} into project {ProjectID}"); var converter = new TAGFileConverter(); using (var fs = new MemoryStream(x.TagFileContent)) { converter.Execute(fs); Log.LogInformation( $"#Progress# TAG file {x.FileName} generated {converter.ProcessedCellPassCount} cell passes from {converter.ProcessedEpochCount} epochs"); } return (x, converter); })).ToList(); // Pick off the tasks as they complete and pass the results into the integrator processor while (tagFileConversions.Count > 0) { async Task<(ProcessTAGFileRequestFileItem, TAGFileConverter)> nextTaskCompleted() { var theTask = await Task.WhenAny(tagFileConversions); tagFileConversions.Remove(theTask); return await theTask; } var convertedTask = nextTaskCompleted(); var converter = convertedTask.Result.Item2; var TAGFile = convertedTask.Result.Item1; try { converter.SiteModel.ID = ProjectID; converter.Machine.ID = AssetID; converter.Machine.IsJohnDoeMachine = TAGFile.IsJohnDoe; integrator.AddTaskToProcessList(converter.SiteModel, ProjectID, converter.Machine, AssetID, converter.SiteModelGridAggregator, converter.ProcessedCellPassCount, converter.MachineTargetValueChangesAggregator); if (++batchCount >= batchSize) { worker.ProcessTask(ProcessedTasks); batchCount = 0; } response.Results.Add(new ProcessTAGFileResponseItem { FileName = TAGFile.FileName, Success = converter.ReadResult == TAGReadResult.NoError, ReadResult = converter.ReadResult }); } catch (Exception E) { response.Results.Add(new ProcessTAGFileResponseItem { FileName = TAGFile.FileName, Success = false, Exception = E.ToString() }); } } if (batchCount > 0) { worker.ProcessTask(ProcessedTasks); } } finally { worker.CompleteTaskProcessing(); } Log.LogInformation($"#Progress# Completed task based conversion of TAG files into project {ProjectID}"); return response; } */ public static ProcessTAGFileResponse Execute(Guid projectId, IEnumerable<ProcessTAGFileRequestFileItem> tagFiles) { var tagFilesArray = tagFiles.ToArray(); // Enumerate collection just once if (tagFilesArray.Length == 0) { // Send back a response with an empty list return new ProcessTAGFileResponse { Results = new List<IProcessTAGFileResponseItem>() }; } _log.LogInformation($"ProcessTAGFileResponse.Execute. Processing {tagFilesArray.Length} TAG files into project {projectId}"); var response = new ProcessTAGFileResponse(); // Create the machinery responsible for tracking tasks and integrating them into the database var integrator = new AggregatedDataIntegrator(); var worker = new AggregatedDataIntegratorWorker(integrator.TasksToProcess, projectId); var processedTasks = new List<AggregatedDataIntegratorTask>(); // Create the site model and machine etc to aggregate the processed TAG file into // Note: This creates these elements within the project itself, not just class instances... // SiteModel siteModel = SiteModels.SiteModels.Instance(StorageMutability.Mutable).GetSiteModel(ProjectUID, true); // Machine machine = new Machine(null, "TestName", "TestHardwareID", 0, 0, Guid.NewGuid(), 0, false); // Assertion: All TAG files relate to the same machine in the same project // Progressively process each TAG file into the same intermediary site model before integrating that intermediary model // into the primary persistent model. // TODO: Failure of a single TAG file may result in contamination of the results of previous processed TAG files requiring exclusion of the TAG file in question and reprocessing of the list _log.LogInformation($"#Progress# Initiating task based conversion of TAG files into project {projectId}"); try { using (var commonConverter = new TAGFileConverter(projectId)) { foreach (var tagFile in tagFilesArray) { using var fs = new MemoryStream(tagFile.TagFileContent); try { _log.LogInformation($"About to read file {tagFile.FileName} with origin source {tagFile.OriginSource}"); var readResult = tagFile.OriginSource switch { TAGFileOriginSource.VolvoMachineAssistEarthworksCSV => commonConverter.ExecuteVolvoEarthworksCSVFile(tagFile.FileName, fs, tagFile.AssetId, tagFile.IsJohnDoe), TAGFileOriginSource.VolvoMachineAssistCompactionCSV => false, TAGFileOriginSource.LegacyTAGFileSource => commonConverter.ExecuteLegacyTAGFile(tagFile.FileName, fs, tagFile.AssetId, tagFile.IsJohnDoe), _ => throw new NotImplementedException("Unsupported commonConverter origin source"), }; response.Results.Add(new ProcessTAGFileResponseItem { FileName = tagFile.FileName, AssetUid = tagFile.AssetId, ReadResult = commonConverter.ReadResult, Success = commonConverter.ReadResult == TAGReadResult.NoError && readResult == true, SubmissionFlags = tagFile.SubmissionFlags, OriginSource = tagFile.OriginSource }); _log.LogInformation( $"#Progress# [CommonConverter] TAG file {tagFile.FileName} generated {commonConverter.ProcessedCellPassCount} cell passes from {commonConverter.ProcessedEpochCount} epochs for asset {tagFile.AssetId} machinetype {commonConverter.Machine?.MachineType.ToString() ?? "null"} with result {commonConverter.ReadResult}"); } catch (Exception e) { _log.LogError(e, $"Processing of TAG file {tagFile.FileName} failed with exception {e.Message}"); response.Results.Add(new ProcessTAGFileResponseItem { FileName = tagFile.FileName, Success = false, Exception = e.Message, ReadResult = commonConverter.ReadResult, SubmissionFlags = tagFile.SubmissionFlags, OriginSource = tagFile.OriginSource }); } } commonConverter.SiteModel.ID = projectId; integrator.AddTaskToProcessList(commonConverter.SiteModel, projectId, commonConverter.Machines, commonConverter.SiteModelGridAggregator, commonConverter.ProcessedCellPassCount, commonConverter.MachinesTargetValueChangesAggregator); } worker.ProcessTask(processedTasks, tagFilesArray.Length); } finally { worker.CompleteTaskProcessing(); } _log.LogInformation($"#Progress# Completed task based conversion of TAG files into project {projectId}"); return response; }