/// <summary>
/// Integrates the cell passes processed from TAG files into sub grids within the live site model
/// </summary>
/// <param name="siteModelFromDatamodel">The site model to perform the change notifications for</param>
/// <param name="task">The 'seed' task used as a hold all for aggregated machines</param>
/// <param name="subGridIntegrator">The integrator to use to insert the new cell passes into the live site model</param>
/// <param name="groupedAggregatedCellPasses">The set of all cell passes from all TAG files, grouped in to a single intermediary site model</param>
/// <param name="numTagFilesRepresented">The number of TAG files represented in the data set being integrated</param>
/// <param name="totalPassCountInAggregation">The sum total number of cell passes integrated in the live site model</param>
private bool IntegrateCellPassesIntoLiveSiteModel(ISiteModel siteModelFromDatamodel,
AggregatedDataIntegratorTask task,
SubGridIntegrator subGridIntegrator,
ISubGridTree groupedAggregatedCellPasses,
int numTagFilesRepresented,
out long totalPassCountInAggregation)
{
_log.LogInformation($"Aggregation Task Process --> Labeling aggregated cell pass with correct machine ID for {siteModelFromDatamodel.ID}");
totalPassCountInAggregation = 0;
// This is a dirty map for the leaf sub grids and is stored as a bitmap grid
// with one level fewer that the sub grid tree it is representing, and
// with cells the size of the leaf sub grids themselves. As the cell coordinates
// we have been given are with respect to the sub grid, we must transform them
// into coordinates relevant to the dirty bitmap sub grid tree.
_workingModelUpdateMap = new SubGridTreeSubGridExistenceBitMask
{
CellSize = SubGridTreeConsts.SubGridTreeDimension * siteModelFromDatamodel.CellSize,
ID = siteModelFromDatamodel.ID
};
// Integrate the cell pass data into the main site model and commit each sub grid as it is updated
// ... first relabel the passes with the machine IDs from the persistent datamodel
// Compute the vector of internal site model machine indexes between the intermediary site model constructed from the TAG files,
// and the persistent site model the data us being processed into
(short taskInternalMachineIndex, short datamodelInternalMachineIndex)[] internalMachineIndexMap = task.IntermediaryTargetMachines
/// <summary>
/// Group all provided sub grid trees containing TAG file swathing results into a single aggregate sub grid tree
/// </summary>
/// <param name="processedTasks">The collated set tasks ready for aggregation</param>
/// <param name="task">The 'seed' task used as a template for locating matching tasks</param>
/// <returns>The sub grid tree containing all processed TAG file cell passes ready for aggregation</returns>
private IServerSubGridTree GroupSwathedSubGridTrees(List <AggregatedDataIntegratorTask> processedTasks,
AggregatedDataIntegratorTask task)
{
// Use the grouped sub grid tree integrator to assemble a single aggregate tree from the set of trees in the processed tasks
_log.LogDebug($"Aggregation Task Process --> Integrate {processedTasks.Count} cell pass trees");
IServerSubGridTree groupedAggregatedCellPasses;
if (processedTasks.Count > 1)
{
var subGridTreeIntegrator = new GroupedSubGridTreeIntegrator
{
Trees = processedTasks
.Where(t => t.AggregatedCellPassCount > 0)
.Select(t => (t.AggregatedCellPasses, DateTime.MinValue, DateTime.MaxValue))
.ToList()
};
// Assign the new grid into Task to represent the spatial aggregation of all of the tasks aggregated cell passes
groupedAggregatedCellPasses = subGridTreeIntegrator.IntegrateSubGridTreeGroup();
}
else
{
groupedAggregatedCellPasses = task.AggregatedCellPasses;
}
#if CELLDEBUG
groupedAggregatedCellPasses?.ScanAllSubGrids(leaf =>
{
foreach (var segment in ((ServerSubGridTreeLeaf)leaf).Cells.PassesData.Items)
{
foreach (var cell in segment.PassesData.GetState())
{
cell.CheckPassesAreInCorrectTimeOrder("Cell passes not in correct order at point groupedAggregatedCellPasses is determined");
}
}
return(true);
});
#endif
// Discard all the aggregated cell pass models for the tasks being processed as they have now been aggregated into
// the model represented by groupedAggregatedCellPasses
_log.LogDebug("Aggregation Task Process --> Clean up cell pass trees");
processedTasks.ForEach(x =>
{
if (x.AggregatedCellPasses != task.AggregatedCellPasses)
{
x.AggregatedCellPasses.Dispose();
x.AggregatedCellPasses = null;
}
});
return(groupedAggregatedCellPasses);
}
/// <summary>
///
/// </summary>
/// <param name="task">The 'seed' task used as a template for locating matching tasks</param>
private ISiteModel ObtainSiteModel(AggregatedDataIntegratorTask task)
{
// Note: This request for the SiteModel specifically asks for the mutable grid SiteModel
var siteModelFromDatamodel = DIContext.Obtain <ISiteModels>().GetSiteModel(task.PersistedTargetSiteModelID, true);
siteModelFromDatamodel?.SetStorageRepresentationToSupply(StorageMutability.Mutable);
return(siteModelFromDatamodel);
}
/// <summary>
/// Aggregates all the machine events from the processed TAG files into a single collection, and
/// maintains last known values for location, hardware ID and machine type
/// </summary>
/// <param name="processedTasks">The collated set tasks ready for aggregation</param>
/// <param name="task">The 'seed' task used as a hold all for aggregated machines</param>
private EventIntegrator AggregateAllMachineEvents(List <AggregatedDataIntegratorTask> processedTasks,
AggregatedDataIntegratorTask task)
{
_log.LogDebug("Aggregation Task Process --> Aggregate machine events");
var eventIntegrator = new EventIntegrator();
// Iterate through the tasks to integrate the machine events
for (var i = 1; i < processedTasks.Count; i++) // Zeroth item in the list is Task
{
var processedTask = processedTasks[i];
// 'Include' the extents etc of each site model being merged into 'task' into its extents and design change events
task.IntermediaryTargetSiteModel.Include(processedTask.IntermediaryTargetSiteModel);
// Iterate over all the machine events collected in the task
foreach (var machine in processedTask.IntermediaryTargetMachines)
{
// Integrate the machine events
eventIntegrator.IntegrateMachineEvents
(processedTask.AggregatedMachineEvents[machine.InternalSiteModelMachineIndex],
task.AggregatedMachineEvents[machine.InternalSiteModelMachineIndex], false,
processedTask.IntermediaryTargetSiteModel, task.IntermediaryTargetSiteModel);
//Update current DateTime with the latest one
if (machine.LastKnownPositionTimeStamp.CompareTo(machine.LastKnownPositionTimeStamp) == -1)
{
machine.LastKnownPositionTimeStamp = machine.LastKnownPositionTimeStamp;
machine.LastKnownX = machine.LastKnownX;
machine.LastKnownY = machine.LastKnownY;
}
if (string.IsNullOrEmpty(machine.MachineHardwareID))
{
machine.MachineHardwareID = machine.MachineHardwareID;
}
if (machine.MachineType == 0)
{
machine.MachineType = machine.MachineType;
}
}
}
return(eventIntegrator);
}
/// <summary>
/// Assembles a group of TAG files to be processed in a single operation
/// </summary>
/// <param name="processedTasks">The collection of tasks being collated ready for aggregation</param>
/// <param name="task">The 'seed' task used as a template for locating matching tasks</param>
private void AssembleGroupedTagFiles(List <AggregatedDataIntegratorTask> processedTasks,
AggregatedDataIntegratorTask task)
{
_log.LogInformation("Aggregation Task Process --> Filter tasks to aggregate");
// Populate the tasks to process list with the aggregations that will be
// processed at this time. These tasks are also removed from the main task
// list to allow the TAG file processors to prepare additional TAG files
// while this set is being integrated into the model.
while (processedTasks.Count < MaxMappedTagFilesToProcessPerAggregationEpoch &&
_tasksToProcess.TryDequeue(out var taskToProcess))
{
processedTasks.Add(taskToProcess);
}
_log.LogInformation($"Aggregation Task Process --> Integrating {processedTasks.Count} TAG file processing tasks for project {task.PersistedTargetSiteModelID}");
}
// AddTaskToProcessList adds a task to the processing queue for the task
// processor. This is a thread safe call, multiple threads may safely add
// tasks to the list in a concurrent fashion if required.
// Each task added to the process list represents a tag file that has been
// processed
public void AddTaskToProcessList(ISiteModel transientSiteModel,
Guid persistentSiteModelId,
IMachinesList transientMachines,
IServerSubGridTree aggregatedCellPasses,
int aggregatedCellPassCount,
IMachinesProductionEventLists aggregatedMachineEvents)
{
var newTask = new AggregatedDataIntegratorTask
{
IntermediaryTargetSiteModel = transientSiteModel,
PersistedTargetSiteModelID = persistentSiteModelId,
IntermediaryTargetMachines = transientMachines,
AggregatedCellPasses = aggregatedCellPasses,
AggregatedMachineEvents = aggregatedMachineEvents,
AggregatedCellPassCount = aggregatedCellPassCount
};
IncrementOutstandingCellPasses(aggregatedCellPassCount);
System.Threading.Interlocked.Add(ref _totalCellPassesProcessed, aggregatedCellPassCount);
TasksToProcess.Enqueue(newTask);
System.Threading.Interlocked.Increment(ref _pendingFilesToBeProcessedCount);
}
/// <summary>
/// Integrates all the machine events from the processed TAG files into the matching machines in the live site model
/// </summary>
/// <param name="siteModelFromDatamodel">The persistent site model the events will be integrated into</param>
/// <param name="task">The 'seed' task used as a hold all for aggregated machines</param>
/// <param name="eventIntegrator">The event integrator to perform the mechanical insertion of the new events</param>
private bool IntegrateMachineEventsIntoLiveSiteModel(ISiteModel siteModelFromDatamodel, AggregatedDataIntegratorTask task,
EventIntegrator eventIntegrator)
{
// Perform machine event integration outside of the SiteModel write access interlock as the
// individual event lists have independent exclusive locks event integration uses.
_log.LogDebug($"Aggregation Task Process --> Integrating machine events into the live site model for {task.IntermediaryTargetMachines.Count} intermediary machines");
// Iterate over all the machine events collected in the task
foreach (var machineFromTask in task.IntermediaryTargetMachines)
{
var machineFromDatamodel = siteModelFromDatamodel.Machines.Locate(machineFromTask.ID, machineFromTask.Name, machineFromTask.IsJohnDoeMachine);
var siteModelMachineTargetValues = siteModelFromDatamodel.MachinesTargetValues[machineFromDatamodel.InternalSiteModelMachineIndex];
eventIntegrator.IntegrateMachineEvents(task.AggregatedMachineEvents[machineFromTask.InternalSiteModelMachineIndex],
siteModelMachineTargetValues, true, task.IntermediaryTargetSiteModel, siteModelFromDatamodel);
// Integrate the machine events into the main site model. This requires the
// site model interlock as aspects of the site model state (machine) are being changed.
if (siteModelMachineTargetValues != null)
{
//Update machine last known value (events) from integrated model before saving
var comparison = machineFromDatamodel.LastKnownPositionTimeStamp.CompareTo(machineFromDatamodel.LastKnownPositionTimeStamp);
if (comparison == -1)
{
machineFromDatamodel.LastKnownDesignName = siteModelFromDatamodel.SiteModelMachineDesigns[siteModelMachineTargetValues.MachineDesignNameIDStateEvents.LastStateValue()].Name;
machineFromDatamodel.LastKnownLayerId = siteModelMachineTargetValues.LayerIDStateEvents.Count() > 0 ? siteModelMachineTargetValues.LayerIDStateEvents.LastStateValue() : (ushort)0;
machineFromDatamodel.LastKnownPositionTimeStamp = machineFromDatamodel.LastKnownPositionTimeStamp;
machineFromDatamodel.LastKnownX = machineFromDatamodel.LastKnownX;
machineFromDatamodel.LastKnownY = machineFromDatamodel.LastKnownY;
}
}
else
{
_log.LogError("SiteModelMachineTargetValues not located in aggregate machine events integrator");
return(false);
}
// Use the synchronous command to save the machine events to the persistent store into the deferred (asynchronous model)
siteModelMachineTargetValues.SaveMachineEventsToPersistentStore(_storageProxyMutable);
}
return(true);
}
/// <summary>
/// Creates new or updates existing machines in the site model ready to receive the aggregated machine events
/// </summary>
/// <param name="siteModelFromDatamodel">The persistent site model the events will be integrated into</param>
/// <param name="task">The 'seed' task used as a hold all for aggregated machines</param>
private void CreateOrUpdateSiteModelMachines(ISiteModel siteModelFromDatamodel, AggregatedDataIntegratorTask task)
{
// Integrate the items present in the 'IntermediaryTargetSiteModel' into the real site model
// read from the datamodel file itself, then synchronously write it to the DataModel
_log.LogDebug("Aggregation Task Process --> Creating and updating machines in the live site model");
// 'Include' the extents etc of the 'task' each site model being merged into the persistent database
siteModelFromDatamodel.Include(task.IntermediaryTargetSiteModel);
// Iterate over all the machine events collected in the task
foreach (var machineFromTask in task.IntermediaryTargetMachines)
{
// Need to locate or create a matching machine in the site model.
var machineFromDatamodel = siteModelFromDatamodel.Machines.Locate(machineFromTask.ID, machineFromTask.Name, machineFromTask.IsJohnDoeMachine);
// Log.LogInformation($"Selecting machine: PersistedTargetMachineID={task.PersistedTargetMachineID}, IsJohnDoe?:{task.IntermediaryTargetMachine.IsJohnDoeMachine}, Result: {machineFromDatamodel}");
if (machineFromDatamodel == null)
{
_log.LogInformation($"Creating new machine: Name={machineFromTask.Name}, MachineHardwareID={machineFromTask.MachineHardwareID}, MachineType={machineFromTask.MachineType} DeviceType={machineFromTask.DeviceType} IsJohnDoe ?:{machineFromTask.IsJohnDoeMachine}, ID: {machineFromTask.ID}");
machineFromDatamodel = siteModelFromDatamodel.Machines.CreateNew(machineFromTask.Name,
machineFromTask.MachineHardwareID,
machineFromTask.MachineType,
machineFromTask.DeviceType,
machineFromTask.IsJohnDoeMachine,
machineFromTask.ID);
machineFromDatamodel.Assign(machineFromTask);
}
// Update the internal name of the machine with the machine name from the TAG file
if (machineFromTask.Name != "" && machineFromDatamodel.Name != machineFromTask.Name)
{
machineFromDatamodel.Name = machineFromTask.Name;
}
// Update the internal type of the machine with the machine type from the TAG file
// if the existing internal machine type is zero then
if (machineFromTask.MachineType != 0 && machineFromDatamodel.MachineType == 0)
{
machineFromDatamodel.MachineType = machineFromTask.MachineType;
}
// If the machine target values can't be found then create them
var siteModelMachineTargetValues = siteModelFromDatamodel.MachinesTargetValues[machineFromDatamodel.InternalSiteModelMachineIndex];
if (siteModelMachineTargetValues == null)
{
siteModelFromDatamodel.MachinesTargetValues.Add(new ProductionEventLists(siteModelFromDatamodel, machineFromDatamodel.InternalSiteModelMachineIndex));
}
}
}
