/*
* /// <summary>
* /// The version number to stamp spatial data elements with generated by this worker.
* /// Commented out pending requirement for request based consistency requiring a sub grid version map
* /// </summary>
* //private long EpochVersion { get; } = DateTime.UtcNow.Ticks;
*/
/// <summary>
/// Constructor the initializes state ready for integration. Not that two storage proxies are provided - one to store sub grid dictionary/headers, and one to store segments
/// so that they can be committed to the persistent store safely (ie: segments first, then headers).
/// </summary>
/// <param name="source">The sub grid tree from which information is being integrated</param>
/// <param name="siteModel">The site model representing the target sub grid tree</param>
/// <param name="target">The sub grid tree into which the data from the source sub grid tree is integrated</param>
/// <param name="storageProxySubGrids">The storage proxy providing storage semantics for persisting integration results relating to sub grids (specifically the header/dictionary representation)</param>
/// <param name="storageProxySubGridSegments">The storage proxy providing storage semantics for persisting integration results relating to sub grid segments</param>
public SubGridIntegrator(IServerSubGridTree source, ISiteModel siteModel, IServerSubGridTree target,
IStorageProxy storageProxySubGrids, IStorageProxy storageProxySubGridSegments)
{
_source = source;
_siteModel = siteModel;
_target = target;
_storageProxySubGrids = storageProxySubGrids;
_storageProxySubGridSegments = storageProxySubGridSegments;
}
public bool CompactionDataSupportedByMachine => true; // Todo: Need to wire this into subscriptions
public SwatherBase(TAGProcessorBase processor,
IProductionEventLists machineTargetValueChanges,
ISiteModel siteModel,
IServerSubGridTree grid,
Fence interpolationFence)
{
Processor = processor;
MachineTargetValueChanges = machineTargetValueChanges;
SiteModel = siteModel;
Grid = grid;
InterpolationFence = interpolationFence;
}
/// <summary>
/// Primary constructor for the TAGProcessor. The arguments passed to it are:
/// 1. The target SiteModel which is intended to be the recipient of the TAG information processed
/// 2. The target Machine in the site model which recorded the TAG information
/// 3. The event lists related to the target machine in the target site model
/// 4. A subgrid tree representing the aggregator for all the spatial cell pass information processed
/// from the TAG information as an independent entity.
/// 5. A set of event lists representing the aggregator for all the machine events for the target machine
/// in the target site model that were processed from the TAG information as a separate entity.
/// </summary>
/// <param name="targetSiteModel"></param>
/// <param name="targetMachine"></param>
/// <param name="siteModelGridAggregator"></param>
/// <param name="machineTargetValueChangesAggregator"></param>
public TAGProcessor(ISiteModel targetSiteModel,
IMachine targetMachine,
IServerSubGridTree siteModelGridAggregator,
ProductionEventLists machineTargetValueChangesAggregator) : this()
{
SiteModel = targetSiteModel;
Machine = targetMachine;
SiteModelGridAggregator = siteModelGridAggregator;
MachineTargetValueChangesAggregator = machineTargetValueChangesAggregator;
// MachineTargetValueChangesAggregator.MarkAllEventListsAsInMemoryOnly;
}
/// <summary>
/// Locates a sub grid in within this site model. If the sub grid cannot be found it will be created.
/// If requested from an immutable grid context, the result of this call should be considered as an immutable
/// copy of the requested data that is valid for the duration the request holds a reference to it. Updates
/// to sub grids in this data model from ingest processing and other operations performed in mutable contexts
/// can occur while this request is in process, but will not affected the immutable copy initially requested.
/// If requested from a mutable grid context the calling context is responsible for ensuring serialized write access
/// to the data elements being requested.
/// </summary>
// ReSharper disable once MemberCanBePrivate.Global
public IServerLeafSubGrid LocateOrCreateSubGrid(IServerSubGridTree grid, int cellX, int cellY)
{
var result = SubGridUtilities.LocateSubGridContaining(
_storageProxySubGrids,
grid,
// DataStoreInstance.GridDataCache,
cellX, cellY,
grid.NumLevels,
false, true) as IServerLeafSubGrid;
// Ensure the cells and segment directory are initialized if this is a new sub grid
if (result != null)
{
// By definition, any new sub grid we create here is dirty, even if we
// ultimately do not add any cell passes to it. This is necessary to
// encourage even otherwise empty sub grids to be persisted to disk if
// they have been created, but never populated with cell passes.
// The sub grid persistent layer may implement a rule that no empty
// sub grids are saved to disk if this becomes an issue...
result.SetDirty();
result.AllocateLeafFullPassStacks();
if (result.Directory.SegmentDirectory.Count == 0)
{
result.Cells.SelectSegment(Consts.MIN_DATETIME_AS_UTC);
}
if (result.Cells == null)
{
_log.LogCritical($"LocateSubGridContaining returned a sub grid {result.Moniker()} with no allocated cells");
}
else if (result.Directory.SegmentDirectory.Count == 0)
{
_log.LogCritical($"LocateSubGridContaining returned a sub grid {result.Moniker()} with no segments in its directory");
}
}
else
{
_log.LogCritical($"LocateSubGridContaining failed to return a sub grid (CellX/Y={cellX}/{cellY})");
}
return(result);
}
// AddTaskToProcessList adds a task to the processing queue for the task
// processor. This is a thread safe call, multiple threads may safely add
// tasks to the list in a concurrent fashion if required.
// Each task added to the process list represents a tag file that has been
// processed
public void AddTaskToProcessList(ISiteModel transientSiteModel,
Guid persistentSiteModelId,
IMachinesList transientMachines,
IServerSubGridTree aggregatedCellPasses,
int aggregatedCellPassCount,
IMachinesProductionEventLists aggregatedMachineEvents)
{
var newTask = new AggregatedDataIntegratorTask
{
IntermediaryTargetSiteModel = transientSiteModel,
PersistedTargetSiteModelID = persistentSiteModelId,
IntermediaryTargetMachines = transientMachines,
AggregatedCellPasses = aggregatedCellPasses,
AggregatedMachineEvents = aggregatedMachineEvents,
AggregatedCellPassCount = aggregatedCellPassCount
};
IncrementOutstandingCellPasses(aggregatedCellPassCount);
System.Threading.Interlocked.Add(ref _totalCellPassesProcessed, aggregatedCellPassCount);
TasksToProcess.Enqueue(newTask);
System.Threading.Interlocked.Increment(ref _pendingFilesToBeProcessedCount);
}
/// <summary>
/// Locates the sub grid in the sub grid tree that contains the cell identified by CellX and CellY in the global
/// sub grid tree cell address space. The tree level for the sub grid returned is specified in Level.
/// </summary>
public static ISubGrid LocateSubGridContaining(IStorageProxy storageProxyForSubGrids,
IServerSubGridTree forSubGridTree,
//const GridDataCache : TICDataStoreCache;
int cellX,
int cellY,
byte level,
bool lookInCacheOnly,
bool acceptSpeculativeReadFailure)
{
IServerLeafSubGrid leafSubGrid = null;
var createdANewSubGrid = false;
ISubGrid result = null;
if (forSubGridTree == null)
{
throw new TRexSubGridProcessingException($"Sub grid tree null in {nameof(LocateSubGridContaining)}");
}
// Note: Sub grid tree specific interlocks are no longer used. The tree now internally
// manages fine grained locks across structurally mutating activities such as node/leaf
// sub grid addition and reading content from the persistent store.
// First check to see if the requested cell is present in a leaf sub grid
var subGrid = forSubGridTree.LocateClosestSubGridContaining(cellX, cellY, level);
if (subGrid == null) // Something bad happened
{
_log.LogWarning($"Failed to locate sub grid at {cellX}:{cellY}, level {level}, data model ID:{forSubGridTree.ID}");
return(null);
}
if (!subGrid.IsLeafSubGrid() && !lookInCacheOnly && level == forSubGridTree.NumLevels)
{
if (forSubGridTree.CachingStrategy == ServerSubGridTreeCachingStrategy.CacheSubGridsInTree)
{
// Create the leaf sub grid that will be used to read in the sub grid from the disk.
// In the case where the sub grid isn't present on the disk this reference will be destroyed
subGrid = forSubGridTree.ConstructPathToCell(cellX, cellY, Types.SubGridPathConstructionType.CreateLeaf);
}
else if (forSubGridTree.CachingStrategy == ServerSubGridTreeCachingStrategy.CacheSubGridsInIgniteGridCache)
{
// Create the leaf sub grid without constructing elements in the grid to represent it other than the
// path in the tree to the parent of the sub grid.
// Note: Setting owner and parent relationship from the sub grid to the tree in this fashion permits
// business logic in th sub grid that require knowledge of it parent and owner relationships to function
// correctly while not including a reference to the sub grid from the tree.
subGrid = forSubGridTree.CreateNewSubGrid(forSubGridTree.NumLevels);
subGrid.SetAbsoluteOriginPosition(cellX & ~SubGridTreeConsts.SubGridLocalKeyMask, cellY & ~SubGridTreeConsts.SubGridLocalKeyMask);
subGrid.Owner = forSubGridTree;
subGrid.Parent = forSubGridTree.ConstructPathToCell(cellX, cellY, Types.SubGridPathConstructionType.CreatePathToLeaf);
}
if (subGrid != null)
{
createdANewSubGrid = true;
}
else
{
_log.LogError($"Failed to create leaf sub grid in LocateSubGridContaining for sub grid at {cellX}x{cellY}");
return(null);
}
}
if (subGrid.IsLeafSubGrid())
{
leafSubGrid = subGrid as IServerLeafSubGrid;
}
if (leafSubGrid == null) // Something bad happened
{
_log.LogError($"Sub grid request result for {cellX}:{cellY} is not a leaf sub grid, it is a {subGrid.GetType().Name}.");
return(null);
}
if (!createdANewSubGrid)
{
if (lookInCacheOnly)
{
if (subGrid.Level == level)
{
return(subGrid);
}
// If the returned sub grid is a leaf sub grid then it was already present in the
// cache. If the level of the returned sub grid matches the request level parameter
// then there is nothing more to do here.
if (subGrid.IsLeafSubGrid() &&
((leafSubGrid.HasSubGridDirectoryDetails || leafSubGrid.Dirty) &&
leafSubGrid.HasAllCellPasses() && leafSubGrid.HasLatestData()) ||
(!subGrid.IsLeafSubGrid() && subGrid.Level == level))
{
return(subGrid);
}
}
}
if ((!leafSubGrid.HasSubGridDirectoryDetails && !leafSubGrid.Dirty) ||
!(leafSubGrid.HasAllCellPasses() && leafSubGrid.HasLatestData()))
{
// The requested cell is either not present in the sub grid tree (cache),
// or it is residing on disk, and a newly created sub grid has been constructed
// to contain the data read from disk.
// The underlying assumption is that this method is only called if the caller knows
// that the sub grid exists in the sub grid tree (this is known via the sub grid existence
// map available to the caller). In cases where eventual consistency
// may mean that a sub grid was removed from the sub grid tree since the caller retrieved
// its copy of the sub grid existence map this function will fail gracefully with a null sub grid.
// The exception to this rule is the tag file processor service which may speculatively
// attempt to read a sub grid that doesn't exist.
// This is a different approach to desktop systems where the individual node sub grids
// contain mini existence maps for the sub grids below them.
if (forSubGridTree.LoadLeafSubGrid(storageProxyForSubGrids,
new SubGridCellAddress(cellX, cellY),
true, true,
leafSubGrid))
{
// We've loaded it - get the reference to the new sub grid and return it
result = leafSubGrid;
}
else
{
// The sub grid could not be loaded. This is likely due to it not ever existing
// in the model, or it may have been deleted. Failure here does not necessarily
// constitute evidence of corruption in the data model. Examination of the
// spatial existence map in conjunction with the requested sub grid index is
// required to determine that. Advise the caller nothing was read by sending back
// a null sub grid reference.
// The failed sub grid is not proactively deleted and will remain so the normal cache
// expiry mechanism can remove it in its normal operations
if (acceptSpeculativeReadFailure)
{
// Return the otherwise empty sub grid back to the caller and integrate it into the cache
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Speculative read failure accepted for sub grid {leafSubGrid.Moniker()}. Blank sub grid returned to caller.");
}
result = leafSubGrid;
}
else
{
_log.LogWarning($"Failed to read leaf sub grid {leafSubGrid.Moniker()} in model {forSubGridTree.ID}. Failed sub grid is NOT removed from the tree");
// Empty the sub grid leaf based data to encourage it to be read on a secondary attempt
leafSubGrid.DeAllocateLeafFullPassStacks();
leafSubGrid.DeAllocateLeafLatestPassGrid();
}
}
}
// Ignite special case - allow Dirty leaf sub grids to be returned
if (result == null)
{
if (leafSubGrid.HasSubGridDirectoryDetails && leafSubGrid.Dirty && leafSubGrid.HasAllCellPasses() && leafSubGrid.HasLatestData())
{
result = leafSubGrid;
}
}
// IGNITE: Last gasp - if the sub grid is in memory and has directory details then just return it
if (result == null && leafSubGrid.HasSubGridDirectoryDetails)
{
result = leafSubGrid;
}
return(result);
}
