public void Notify_EmptyChangeMap(SiteModelChangeMapOrigin origin, SiteModelChangeMapOperation operation)
{
var notifier = new SiteModelChangeMapDeltaNotifier();
// The notifier uses the non-transacted storage proxy:
var proxy = DIContext.Obtain <Func <IStorageProxyCache <ISiteModelChangeBufferQueueKey, ISiteModelChangeBufferQueueItem> > >()();
proxy.Should().NotBeNull();
proxy.Clear();
var projectUid = Guid.NewGuid();
var insertUtc = DateTime.UtcNow;
// Ask the notifier to notify a new item
notifier.Notify(projectUid, insertUtc, new SubGridTreeSubGridExistenceBitMask(), origin, operation);
// Check the new item was placed into the cache
var cachedItem = proxy.Get(new SiteModelChangeBufferQueueKey(projectUid, insertUtc));
cachedItem.Should().NotBeNull();
cachedItem.Operation.Should().Be(operation);
cachedItem.Origin.Should().Be(origin);
var readMap = new SubGridTreeSubGridExistenceBitMask();
readMap.FromBytes(cachedItem.Content);
readMap.CountBits().Should().Be(0);
}
public void Notify_TAGFileDerivedChangeMap_SiteModelsMediatedNotification()
{
// Build a site model from a TAG file and verify there is a change map written to the queue that matches the existence map
// for the newly created model
var tagFiles = new[]
{
Path.Combine(TestHelper.CommonTestDataPath, "TestTAGFile.tag"),
};
var siteModel = DITAGFileAndSubGridRequestsFixture.BuildModel(tagFiles, out _);
// The notifier uses the non-transacted storage proxy:
var proxy = DIContext.Obtain <Func <IStorageProxyCache <ISiteModelChangeBufferQueueKey, ISiteModelChangeBufferQueueItem> > >()();
proxy.Should().NotBeNull();
// Check the new item was placed into the cache
var testProxy = proxy as IStorageProxyCacheTransacted_TestHarness <ISiteModelChangeBufferQueueKey, ISiteModelChangeBufferQueueItem>;
testProxy.GetPendingTransactedWrites().Count.Should().Be(1);
var cachedItem = testProxy.GetPendingTransactedWrites().Values.First();
cachedItem.Should().NotBeNull();
cachedItem.ProjectUID.Should().Be(siteModel.ID);
cachedItem.Operation.Should().Be(SiteModelChangeMapOperation.AddSpatialChanges);
cachedItem.Origin.Should().Be(SiteModelChangeMapOrigin.Ingest);
var readMap = new SubGridTreeSubGridExistenceBitMask();
readMap.FromBytes(cachedItem.Content);
readMap.CountBits().Should().Be(12);
readMap.CountBits().Should().Be(siteModel.ExistenceMap.CountBits());
readMap.ScanAllSetBitsAsSubGridAddresses(x => siteModel.ExistenceMap[x.X >> SubGridTreeConsts.SubGridIndexBitsPerLevel, x.Y >> SubGridTreeConsts.SubGridIndexBitsPerLevel].Should().BeTrue());
}
public SubGridTreeSubGridExistenceBitMask Get(Guid siteModelUid, Guid assetUid)
{
try
{
var cacheItem = _proxyStorageCache.Get(new SiteModelMachineAffinityKey(siteModelUid, assetUid, FileSystemStreamType.SiteModelMachineElevationChangeMap));
var result = new SubGridTreeSubGridExistenceBitMask();
if (cacheItem != null)
{
result.FromBytes(cacheItem.Bytes);
}
return(result);
}
catch (KeyNotFoundException)
{
return(null);
}
}
/// <summary>
/// Handles the situation when TAG file processing or some other activity has modified the attributes of a site model
/// requiring the site model to be reloaded
/// </summary>
public void SiteModelAttributesHaveChanged(ISiteModelAttributesChangedEvent message)
{
var messageAge = DateTime.UtcNow - message.TimeSentUtc;
_log.LogInformation($"Entering attribute change notification processor for project {message.SiteModelID}, change event ID {message.ChangeEventUid}, event message age {messageAge}");
if (messageAge.TotalSeconds > 1.0)
{
_log.LogWarning($"Message age more than 1 second [{messageAge}]");
}
// Site models have immutable characteristics in TRex. Multiple requests may reference the same site model
// concurrently, with no interlocks enforcing access serialization. Any attempt to replace or modify an already loaded
// site model may cause issue with concurrent request accessing that site model.
// THe strategy here is to preserve continued access by concurrent requests to the site model retrieved
// at the time the request was initiated by removing it from the SiteModels cache but not destroying it.
// Once all request based references to the site model have completed the now orphaned site model will be cleaned
// up by the garbage collector. Removal of the site model is interlocked with getting a site model reference
// to ensure no concurrency issues within the underlying cache implementation
// Note: The site model references some elements that may be preserved via the site model factory method that
// accepts an origin site model.
// These elements are:
// 1. ExistenceMap
// 2. Sub grid tree containing cached sub grid data
// 3. Coordinate system
// 4. Designs
// 5. Surveyed Surfaces
// 6. Machines
// 7. Machines target values
// 8. Machines design names
// 9. Proofing runs
// 10. Alignments
// 11. Site model marked for deletion
ISiteModel siteModel;
// Construct a new site model that preserves elements not affected by the notification and replace the existing
// site model reference with it.
lock (_cachedModels)
{
_cachedModels.TryGetValue(message.SiteModelID, out siteModel);
if (siteModel != null && message.SiteModelMarkedForDeletion)
{
// Remove the site model from the cache and exit.
_cachedModels.Remove(message.SiteModelID);
return;
}
// Note: The spatial data grid is highly conserved and never killed in a site model change notification.
var originFlags =
SiteModelOriginConstructionFlags.PreserveGrid
| (!message.ExistenceMapModified ? SiteModelOriginConstructionFlags.PreserveExistenceMap : 0)
| (!message.CsibModified ? SiteModelOriginConstructionFlags.PreserveCsib : 0)
| (!message.DesignsModified ? SiteModelOriginConstructionFlags.PreserveDesigns : 0)
| (!message.SurveyedSurfacesModified ? SiteModelOriginConstructionFlags.PreserveSurveyedSurfaces : 0)
| (!message.MachinesModified ? SiteModelOriginConstructionFlags.PreserveMachines : 0)
| (!message.MachineTargetValuesModified ? SiteModelOriginConstructionFlags.PreserveMachineTargetValues : 0)
| (!message.MachineDesignsModified ? SiteModelOriginConstructionFlags.PreserveMachineDesigns | SiteModelOriginConstructionFlags.PreserveSiteModelDesigns : 0)
| (!message.ProofingRunsModified ? SiteModelOriginConstructionFlags.PreserveProofingRuns : 0)
| (!message.AlignmentsModified ? SiteModelOriginConstructionFlags.PreserveAlignments : 0)
;
_log.LogInformation($"Processing attribute change notification for site model {message.SiteModelID}. Preserved elements are {originFlags}");
if (siteModel != null)
{
// First create a new site model to replace the site model with, requesting certain elements of the existing site model
// to be preserved in the new site model instance.
siteModel = DIContext.Obtain <ISiteModelFactory>().NewSiteModel(siteModel, originFlags);
// Replace the site model reference in the cache with the new site model
_cachedModels[message.SiteModelID] = siteModel;
}
}
// If the notification contains an existence map change mask then all cached sub grid based elements that match the masked sub grids
// need to be evicted from all cached contexts related to this site model. Note: This operation is not performed under a lock as the
// removal operations on the cache are lock free
if (message.ExistenceMapChangeMask != null)
{
// Create and deserialize the sub grid but mask from the message
var mask = new SubGridTreeSubGridExistenceBitMask();
mask.FromBytes(message.ExistenceMapChangeMask);
if (siteModel != null)
{
// Iterate over all leaf sub grids in the mask. For each get the matching node sub grid in siteModel.Grid,
// and remove all sub grid references from that node sub grid matching the bits in the bit mask sub grid
mask.ScanAllSubGrids(leaf =>
{
// Obtain the matching node sub grid in Grid
var node = siteModel.Grid.LocateClosestSubGridContaining
(leaf.OriginX << SubGridTreeConsts.SubGridIndexBitsPerLevel,
leaf.OriginY << SubGridTreeConsts.SubGridIndexBitsPerLevel,
leaf.Level);
// If there are sub grids present in Grid that match the sub grids identified by leaf
// remove the elements identified in leaf from the node sub grid.
if (node != null)
{
((ISubGridTreeLeafBitmapSubGrid)leaf).ForEachSetBit((x, y) => node.SetSubGrid(x, y, null));
}
return(true); // Keep processing leaf sub grids
});
}
// Advise the spatial memory general sub grid result cache of the change so it can invalidate cached derivatives
DIContext.ObtainOptional <ITRexSpatialMemoryCache>()?.InvalidateDueToProductionDataIngest(message.SiteModelID, message.ChangeEventUid, mask);
// Advise any registered site model change map notifier of the changes
DIContext.ObtainOptional <ISiteModelChangeMapDeltaNotifier>()?.Notify(message.SiteModelID, DateTime.UtcNow, mask, SiteModelChangeMapOrigin.Ingest, SiteModelChangeMapOperation.AddSpatialChanges);
}
}
/// <summary>
/// Takes a site model spatial change map and incorporates those changes in the changes for each machine in the
/// site model.
/// Once items are processed they are removed from the change map queue retirement queue.
/// </summary>
private bool ProcessItem(ISiteModelChangeBufferQueueItem item)
{
try
{
if (item == null)
{
_log.LogError("Item supplied to queue processor is null. Aborting");
return(false);
}
if (item.Content == null)
{
_log.LogError("Item supplied to queue processor has no internal content. Aborting");
return(false);
}
var siteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(item.ProjectUID);
if (siteModel == null)
{
_log.LogError($"Site model {item.ProjectUID} does not exist [deleted?]. Aborting");
return(false);
}
// Implement change map integration into machine change maps
// 0. Obtain transaction (will create implicit locks on items)
// 1. Read record for machine
// 2. Integrate new map
// 3. Write record back to store
// 4. Commit transaction
_log.LogInformation($"Processing an item: {item.Operation}, project:{item.ProjectUID}, machine:{item.MachineUid}");
var sw = Stopwatch.StartNew();
switch (item.Operation)
{
case SiteModelChangeMapOperation.AddSpatialChanges: // Add the two of them together...
{
// Add the spatial change to every machine in the site model
foreach (var machine in siteModel.Machines)
{
var l = _changeMapProxy.Lock(item.ProjectUID, machine.ID);
l.Enter();
try
{
var currentMask = _changeMapProxy.Get(item.ProjectUID, machine.ID);
if (currentMask == null)
{
currentMask = new SubGridTreeSubGridExistenceBitMask();
currentMask.SetOp_OR(siteModel.ExistenceMap);
}
// Extract the change map from the item
var updateMask = new SubGridTreeSubGridExistenceBitMask();
updateMask.FromBytes(item.Content);
currentMask.SetOp_OR(updateMask);
_changeMapProxy.Put(item.ProjectUID, machine.ID, currentMask);
}
finally
{
l.Exit();
}
}
break;
}
case SiteModelChangeMapOperation.RemoveSpatialChanges: // Subtract from the change map...
{
var l = _changeMapProxy.Lock(item.ProjectUID, item.MachineUid);
l.Enter();
try
{
// Remove the spatial change only from the machine the made the query
var currentMask = _changeMapProxy.Get(item.ProjectUID, item.MachineUid);
if (currentMask != null)
{
// Extract the change map from the item
var updateMask = new SubGridTreeSubGridExistenceBitMask();
currentMask.SetOp_ANDNOT(updateMask);
_changeMapProxy.Put(item.ProjectUID, item.MachineUid, currentMask);
}
}
finally
{
l.Exit();
}
break;
}
default:
_log.LogError($"Unknown operation encountered: {(int) item.Operation}");
return(false);
}
_log.LogInformation($"Completed processing an item in {sw.Elapsed}: {item.Operation}, project:{item.ProjectUID}, machine:{item.MachineUid}");
return(true);
}
catch (Exception e)
{
_log.LogError(e, "Exception thrown while processing queued items:");
throw;
}
}