/// <summary> /// Integrates the cell passes processed from TAG files into sub grids within the live site model /// </summary> /// <param name="siteModelFromDatamodel">The site model to perform the change notifications for</param> /// <param name="task">The 'seed' task used as a hold all for aggregated machines</param> /// <param name="subGridIntegrator">The integrator to use to insert the new cell passes into the live site model</param> /// <param name="groupedAggregatedCellPasses">The set of all cell passes from all TAG files, grouped in to a single intermediary site model</param> /// <param name="numTagFilesRepresented">The number of TAG files represented in the data set being integrated</param> /// <param name="totalPassCountInAggregation">The sum total number of cell passes integrated in the live site model</param> private bool IntegrateCellPassesIntoLiveSiteModel(ISiteModel siteModelFromDatamodel, AggregatedDataIntegratorTask task, SubGridIntegrator subGridIntegrator, ISubGridTree groupedAggregatedCellPasses, int numTagFilesRepresented, out long totalPassCountInAggregation) { _log.LogInformation($"Aggregation Task Process --> Labeling aggregated cell pass with correct machine ID for {siteModelFromDatamodel.ID}"); totalPassCountInAggregation = 0; // This is a dirty map for the leaf sub grids and is stored as a bitmap grid // with one level fewer that the sub grid tree it is representing, and // with cells the size of the leaf sub grids themselves. As the cell coordinates // we have been given are with respect to the sub grid, we must transform them // into coordinates relevant to the dirty bitmap sub grid tree. _workingModelUpdateMap = new SubGridTreeSubGridExistenceBitMask { CellSize = SubGridTreeConsts.SubGridTreeDimension * siteModelFromDatamodel.CellSize, ID = siteModelFromDatamodel.ID }; // Integrate the cell pass data into the main site model and commit each sub grid as it is updated // ... first relabel the passes with the machine IDs from the persistent datamodel // Compute the vector of internal site model machine indexes between the intermediary site model constructed from the TAG files, // and the persistent site model the data us being processed into (short taskInternalMachineIndex, short datamodelInternalMachineIndex)[] internalMachineIndexMap = task.IntermediaryTargetMachines
/// <summary> /// Group all provided sub grid trees containing TAG file swathing results into a single aggregate sub grid tree /// </summary> /// <param name="processedTasks">The collated set tasks ready for aggregation</param> /// <param name="task">The 'seed' task used as a template for locating matching tasks</param> /// <returns>The sub grid tree containing all processed TAG file cell passes ready for aggregation</returns> private IServerSubGridTree GroupSwathedSubGridTrees(List <AggregatedDataIntegratorTask> processedTasks, AggregatedDataIntegratorTask task) { // Use the grouped sub grid tree integrator to assemble a single aggregate tree from the set of trees in the processed tasks _log.LogDebug($"Aggregation Task Process --> Integrate {processedTasks.Count} cell pass trees"); IServerSubGridTree groupedAggregatedCellPasses; if (processedTasks.Count > 1) { var subGridTreeIntegrator = new GroupedSubGridTreeIntegrator { Trees = processedTasks .Where(t => t.AggregatedCellPassCount > 0) .Select(t => (t.AggregatedCellPasses, DateTime.MinValue, DateTime.MaxValue)) .ToList() }; // Assign the new grid into Task to represent the spatial aggregation of all of the tasks aggregated cell passes groupedAggregatedCellPasses = subGridTreeIntegrator.IntegrateSubGridTreeGroup(); } else { groupedAggregatedCellPasses = task.AggregatedCellPasses; } #if CELLDEBUG groupedAggregatedCellPasses?.ScanAllSubGrids(leaf => { foreach (var segment in ((ServerSubGridTreeLeaf)leaf).Cells.PassesData.Items) { foreach (var cell in segment.PassesData.GetState()) { cell.CheckPassesAreInCorrectTimeOrder("Cell passes not in correct order at point groupedAggregatedCellPasses is determined"); } } return(true); }); #endif // Discard all the aggregated cell pass models for the tasks being processed as they have now been aggregated into // the model represented by groupedAggregatedCellPasses _log.LogDebug("Aggregation Task Process --> Clean up cell pass trees"); processedTasks.ForEach(x => { if (x.AggregatedCellPasses != task.AggregatedCellPasses) { x.AggregatedCellPasses.Dispose(); x.AggregatedCellPasses = null; } }); return(groupedAggregatedCellPasses); }
/// <summary> /// /// </summary> /// <param name="task">The 'seed' task used as a template for locating matching tasks</param> private ISiteModel ObtainSiteModel(AggregatedDataIntegratorTask task) { // Note: This request for the SiteModel specifically asks for the mutable grid SiteModel var siteModelFromDatamodel = DIContext.Obtain <ISiteModels>().GetSiteModel(task.PersistedTargetSiteModelID, true); siteModelFromDatamodel?.SetStorageRepresentationToSupply(StorageMutability.Mutable); return(siteModelFromDatamodel); }
/// <summary> /// Aggregates all the machine events from the processed TAG files into a single collection, and /// maintains last known values for location, hardware ID and machine type /// </summary> /// <param name="processedTasks">The collated set tasks ready for aggregation</param> /// <param name="task">The 'seed' task used as a hold all for aggregated machines</param> private EventIntegrator AggregateAllMachineEvents(List <AggregatedDataIntegratorTask> processedTasks, AggregatedDataIntegratorTask task) { _log.LogDebug("Aggregation Task Process --> Aggregate machine events"); var eventIntegrator = new EventIntegrator(); // Iterate through the tasks to integrate the machine events for (var i = 1; i < processedTasks.Count; i++) // Zeroth item in the list is Task { var processedTask = processedTasks[i]; // 'Include' the extents etc of each site model being merged into 'task' into its extents and design change events task.IntermediaryTargetSiteModel.Include(processedTask.IntermediaryTargetSiteModel); // Iterate over all the machine events collected in the task foreach (var machine in processedTask.IntermediaryTargetMachines) { // Integrate the machine events eventIntegrator.IntegrateMachineEvents (processedTask.AggregatedMachineEvents[machine.InternalSiteModelMachineIndex], task.AggregatedMachineEvents[machine.InternalSiteModelMachineIndex], false, processedTask.IntermediaryTargetSiteModel, task.IntermediaryTargetSiteModel); //Update current DateTime with the latest one if (machine.LastKnownPositionTimeStamp.CompareTo(machine.LastKnownPositionTimeStamp) == -1) { machine.LastKnownPositionTimeStamp = machine.LastKnownPositionTimeStamp; machine.LastKnownX = machine.LastKnownX; machine.LastKnownY = machine.LastKnownY; } if (string.IsNullOrEmpty(machine.MachineHardwareID)) { machine.MachineHardwareID = machine.MachineHardwareID; } if (machine.MachineType == 0) { machine.MachineType = machine.MachineType; } } } return(eventIntegrator); }
/// <summary> /// Assembles a group of TAG files to be processed in a single operation /// </summary> /// <param name="processedTasks">The collection of tasks being collated ready for aggregation</param> /// <param name="task">The 'seed' task used as a template for locating matching tasks</param> private void AssembleGroupedTagFiles(List <AggregatedDataIntegratorTask> processedTasks, AggregatedDataIntegratorTask task) { _log.LogInformation("Aggregation Task Process --> Filter tasks to aggregate"); // Populate the tasks to process list with the aggregations that will be // processed at this time. These tasks are also removed from the main task // list to allow the TAG file processors to prepare additional TAG files // while this set is being integrated into the model. while (processedTasks.Count < MaxMappedTagFilesToProcessPerAggregationEpoch && _tasksToProcess.TryDequeue(out var taskToProcess)) { processedTasks.Add(taskToProcess); } _log.LogInformation($"Aggregation Task Process --> Integrating {processedTasks.Count} TAG file processing tasks for project {task.PersistedTargetSiteModelID}"); }
// AddTaskToProcessList adds a task to the processing queue for the task // processor. This is a thread safe call, multiple threads may safely add // tasks to the list in a concurrent fashion if required. // Each task added to the process list represents a tag file that has been // processed public void AddTaskToProcessList(ISiteModel transientSiteModel, Guid persistentSiteModelId, IMachinesList transientMachines, IServerSubGridTree aggregatedCellPasses, int aggregatedCellPassCount, IMachinesProductionEventLists aggregatedMachineEvents) { var newTask = new AggregatedDataIntegratorTask { IntermediaryTargetSiteModel = transientSiteModel, PersistedTargetSiteModelID = persistentSiteModelId, IntermediaryTargetMachines = transientMachines, AggregatedCellPasses = aggregatedCellPasses, AggregatedMachineEvents = aggregatedMachineEvents, AggregatedCellPassCount = aggregatedCellPassCount }; IncrementOutstandingCellPasses(aggregatedCellPassCount); System.Threading.Interlocked.Add(ref _totalCellPassesProcessed, aggregatedCellPassCount); TasksToProcess.Enqueue(newTask); System.Threading.Interlocked.Increment(ref _pendingFilesToBeProcessedCount); }
/// <summary> /// Integrates all the machine events from the processed TAG files into the matching machines in the live site model /// </summary> /// <param name="siteModelFromDatamodel">The persistent site model the events will be integrated into</param> /// <param name="task">The 'seed' task used as a hold all for aggregated machines</param> /// <param name="eventIntegrator">The event integrator to perform the mechanical insertion of the new events</param> private bool IntegrateMachineEventsIntoLiveSiteModel(ISiteModel siteModelFromDatamodel, AggregatedDataIntegratorTask task, EventIntegrator eventIntegrator) { // Perform machine event integration outside of the SiteModel write access interlock as the // individual event lists have independent exclusive locks event integration uses. _log.LogDebug($"Aggregation Task Process --> Integrating machine events into the live site model for {task.IntermediaryTargetMachines.Count} intermediary machines"); // Iterate over all the machine events collected in the task foreach (var machineFromTask in task.IntermediaryTargetMachines) { var machineFromDatamodel = siteModelFromDatamodel.Machines.Locate(machineFromTask.ID, machineFromTask.Name, machineFromTask.IsJohnDoeMachine); var siteModelMachineTargetValues = siteModelFromDatamodel.MachinesTargetValues[machineFromDatamodel.InternalSiteModelMachineIndex]; eventIntegrator.IntegrateMachineEvents(task.AggregatedMachineEvents[machineFromTask.InternalSiteModelMachineIndex], siteModelMachineTargetValues, true, task.IntermediaryTargetSiteModel, siteModelFromDatamodel); // Integrate the machine events into the main site model. This requires the // site model interlock as aspects of the site model state (machine) are being changed. if (siteModelMachineTargetValues != null) { //Update machine last known value (events) from integrated model before saving var comparison = machineFromDatamodel.LastKnownPositionTimeStamp.CompareTo(machineFromDatamodel.LastKnownPositionTimeStamp); if (comparison == -1) { machineFromDatamodel.LastKnownDesignName = siteModelFromDatamodel.SiteModelMachineDesigns[siteModelMachineTargetValues.MachineDesignNameIDStateEvents.LastStateValue()].Name; machineFromDatamodel.LastKnownLayerId = siteModelMachineTargetValues.LayerIDStateEvents.Count() > 0 ? siteModelMachineTargetValues.LayerIDStateEvents.LastStateValue() : (ushort)0; machineFromDatamodel.LastKnownPositionTimeStamp = machineFromDatamodel.LastKnownPositionTimeStamp; machineFromDatamodel.LastKnownX = machineFromDatamodel.LastKnownX; machineFromDatamodel.LastKnownY = machineFromDatamodel.LastKnownY; } } else { _log.LogError("SiteModelMachineTargetValues not located in aggregate machine events integrator"); return(false); } // Use the synchronous command to save the machine events to the persistent store into the deferred (asynchronous model) siteModelMachineTargetValues.SaveMachineEventsToPersistentStore(_storageProxyMutable); } return(true); }
/// <summary> /// Creates new or updates existing machines in the site model ready to receive the aggregated machine events /// </summary> /// <param name="siteModelFromDatamodel">The persistent site model the events will be integrated into</param> /// <param name="task">The 'seed' task used as a hold all for aggregated machines</param> private void CreateOrUpdateSiteModelMachines(ISiteModel siteModelFromDatamodel, AggregatedDataIntegratorTask task) { // Integrate the items present in the 'IntermediaryTargetSiteModel' into the real site model // read from the datamodel file itself, then synchronously write it to the DataModel _log.LogDebug("Aggregation Task Process --> Creating and updating machines in the live site model"); // 'Include' the extents etc of the 'task' each site model being merged into the persistent database siteModelFromDatamodel.Include(task.IntermediaryTargetSiteModel); // Iterate over all the machine events collected in the task foreach (var machineFromTask in task.IntermediaryTargetMachines) { // Need to locate or create a matching machine in the site model. var machineFromDatamodel = siteModelFromDatamodel.Machines.Locate(machineFromTask.ID, machineFromTask.Name, machineFromTask.IsJohnDoeMachine); // Log.LogInformation($"Selecting machine: PersistedTargetMachineID={task.PersistedTargetMachineID}, IsJohnDoe?:{task.IntermediaryTargetMachine.IsJohnDoeMachine}, Result: {machineFromDatamodel}"); if (machineFromDatamodel == null) { _log.LogInformation($"Creating new machine: Name={machineFromTask.Name}, MachineHardwareID={machineFromTask.MachineHardwareID}, MachineType={machineFromTask.MachineType} DeviceType={machineFromTask.DeviceType} IsJohnDoe ?:{machineFromTask.IsJohnDoeMachine}, ID: {machineFromTask.ID}"); machineFromDatamodel = siteModelFromDatamodel.Machines.CreateNew(machineFromTask.Name, machineFromTask.MachineHardwareID, machineFromTask.MachineType, machineFromTask.DeviceType, machineFromTask.IsJohnDoeMachine, machineFromTask.ID); machineFromDatamodel.Assign(machineFromTask); } // Update the internal name of the machine with the machine name from the TAG file if (machineFromTask.Name != "" && machineFromDatamodel.Name != machineFromTask.Name) { machineFromDatamodel.Name = machineFromTask.Name; } // Update the internal type of the machine with the machine type from the TAG file // if the existing internal machine type is zero then if (machineFromTask.MachineType != 0 && machineFromDatamodel.MachineType == 0) { machineFromDatamodel.MachineType = machineFromTask.MachineType; } // If the machine target values can't be found then create them var siteModelMachineTargetValues = siteModelFromDatamodel.MachinesTargetValues[machineFromDatamodel.InternalSiteModelMachineIndex]; if (siteModelMachineTargetValues == null) { siteModelFromDatamodel.MachinesTargetValues.Add(new ProductionEventLists(siteModelFromDatamodel, machineFromDatamodel.InternalSiteModelMachineIndex)); } } }