public void TestRemoveDesignFromDesignCache()
{
AddApplicationRouting();
var siteModel = DITAGFileAndSubGridRequestsWithIgniteFixture.NewEmptyModel();
var designUid = DITAGFileAndSubGridRequestsWithIgniteFixture.AddDesignToSiteModel(ref siteModel, TestHelper.CommonTestDataPath, TESTFILENAME, true);
var localStorage = Path.Combine(FilePathHelper.GetTempFolderForProject(siteModel.ID), TESTFILENAME);
var designFiles = DIContext.ObtainOptional <IDesignFiles>();
designFiles.NumDesignsInCache().Should().Be(0);
designFiles.Lock(designUid, siteModel, SubGridTreeConsts.DefaultCellSize, out var loadResult);
File.Exists(localStorage).Should().BeTrue();
designFiles.NumDesignsInCache().Should().Be(1);
var design1 = loadResult.Should().Be(DesignLoadResult.Success);
design1.Should().NotBeNull();
var message = new DesignChangedEvent()
{
SiteModelUid = siteModel.ID,
DesignUid = designUid,
DesignRemoved = true,
FileType = ImportedFileType.DesignSurface
};
var listener = new DesignChangedEventListener(TRexGrids.ImmutableGridName());
listener.Invoke(Guid.Empty, message).Should().BeTrue();
File.Exists(localStorage).Should().BeFalse();
designFiles.NumDesignsInCache().Should().Be(0);
}
public void Invoke_WithSiteModels()
{
var message = new DesignChangedEvent();
var listener = new DesignChangedEventListener(TRexGrids.ImmutableGridName());
listener.Invoke(Guid.Empty, message).Should().BeTrue();
}
public void Test_ActivatePersistentGridServer_WaitUntilGridActive()
{
var server = new ActivatePersistentGridServer();
server.Should().NotBeNull();
server.WaitUntilGridActive(TRexGrids.ImmutableGridName()).Should().BeTrue();
}
public void StartListening()
{
var listener = new SiteModelAttributesChangedEventListener(TRexGrids.ImmutableGridName())
{
MessageTopicName = "TestMessageTopic"
};
listener.StartListening();
}
public void StartListening_FailWithNoMessagingFabric()
{
var listener = new DesignChangedEventListener(TRexGrids.ImmutableGridName())
{
MessageTopicName = "TestMessageTopic"
};
listener.StartListening();
}
private bool ActivateImmutable()
{
Log.LogInformation("About to call ActivatePersistentGridServer.Instance().SetGridActive() for Immutable TRex grid");
bool result = ActivatePersistentGridServer.Instance().SetGridActive(TRexGrids.ImmutableGridName());
Log.LogInformation($"Activation process completed: Immutable = {result}");
return(result);
}
public void Invoke_WithSiteModels()
{
var message = new SiteModelAttributesChangedEvent
{
SiteModelID = Guid.NewGuid()
};
var listener = new SiteModelAttributesChangedEventListener(TRexGrids.ImmutableGridName());
listener.Invoke(Guid.NewGuid(), message).Should().BeTrue();
}
/// <summary>
/// Creates an appropriate new Ignite client node depending on the TRex Grid it is being attached to
/// </summary>
/// <param name="gridName"></param>
/// <param name="role"></param>
/// <returns></returns>
public static IgniteServer NewClientNode(string gridName, string role)
{
if (gridName.Equals(TRexGrids.MutableGridName()))
{
return(new MutableClientServer(role));
}
if (gridName.Equals(TRexGrids.ImmutableGridName()))
{
return(new ImmutableClientServer(role));
}
throw new ArgumentException($"{gridName} is an unknown grid to create a client node within.");
}
/// <summary>
/// Creates or obtains a reference to an already created change map file buffer queue
/// </summary>
private void InstantiateCache()
{
var ignite = DIContext.Obtain <ITRexGridFactory>()?.Grid(StorageMutability.Immutable) ??
Ignition.GetIgnite(TRexGrids.ImmutableGridName());
_queueCache = ignite?.GetCache <ISiteModelChangeBufferQueueKey, ISiteModelChangeBufferQueueItem>(
TRexCaches.SiteModelChangeBufferQueueCacheName());
if (_queueCache == null)
{
Log.LogInformation($"Failed to get Ignite cache {TRexCaches.SiteModelChangeBufferQueueCacheName()}");
throw new TRexException("Ignite cache not available");
}
}
public static IIgnite GetIgnite(bool force = false)
{
if (!force)
{
throw new InvalidOperationException("Running Ignite nodes in unit tests is undesirable if possible...");
}
lock (LockObj)
{
if (_ignite != null)
{
return(_ignite);
}
_ignite = DIContext.Obtain <ITRexGridFactory>()?.Grid(TRexGrids.ImmutableGridName(), new IgniteConfiguration());
return(_ignite);
}
}
public void StartTRexGridCacheNode()
{
Log.LogInformation("Creating new Ignite node");
var cfg = new IgniteConfiguration();
ConfigureTRexGrid(cfg);
Log.LogInformation($"Creating new Ignite node for {cfg.IgniteInstanceName}");
try
{
Console.WriteLine($"Creating new Ignite node for {cfg.IgniteInstanceName}");
immutableTRexGrid = DIContext.Obtain <ITRexGridFactory>()?.Grid(TRexGrids.ImmutableGridName(), cfg);
}
finally
{
Log.LogInformation($"Completed creation of new Ignite node: Exists = {immutableTRexGrid != null}, Factory available = {DIContext.Obtain<ITRexGridFactory>() != null}");
}
Log.LogInformation($"Network timeout set to {cfg.NetworkTimeout}");
// Wait until the grid is active
DIContext.Obtain <IActivatePersistentGridServer>().WaitUntilGridActive(TRexGrids.ImmutableGridName());
// Add the immutable Spatial & NonSpatial caches
InstantiateNonSpatialCacheReference();
InstantiateSpatialSubGridDirectoryCacheReference();
InstantiateSpatialSubGridSegmentCacheReference();
InstantiateSiteModelExistenceMapsCacheReference();
InstantiateSiteModelsCacheReference();
InstantiateSiteModelChangeBufferQueueCacheReference();
InstantiateSiteModelMachinesChangeMapsCacheReference();
InstantiateDesignTopologyExistenceMapsCache();
}
public SimpleVolumesRequest_ApplicationService() : base(TRexGrids.ImmutableGridName(), ServerRoles.ASNODE)
{
}
/// <summary>
/// Default no-arg constructor that sets up cluster and compute projections available for use
/// </summary>
public CacheComputePoolRequest() : base(TRexGrids.ImmutableGridName(), ServerRoles.PSNODE)
{
}
public void StopListening()
{
var listener = new DesignChangedEventListener(TRexGrids.ImmutableGridName());
listener.StopListening();
}
/// <summary>
/// Default constructor that configures the request to be sent to the TIN export projection on the immutable data grid
/// </summary>
public TINSurfaceRequest() : base(TRexGrids.ImmutableGridName(), ServerRoles.TIN_SURFACE_EXPORT_ROLE)
{
}
public PatchRequest() : base(TRexGrids.ImmutableGridName(), ServerRoles.ASNODE)
{
}
public PassCountStatisticsRequest_ApplicationService() : base(TRexGrids.ImmutableGridName(), ServerRoles.ASNODE)
{
}
/// <summary>
/// Constructor that creates a new server instance with a set of roles
/// </summary>
/// <param name="roles"></param>
public ImmutableClientServer(string[] roles)
{
if (immutableTRexGrid == null)
{
// Attempt to attach to an already existing Ignite instance
Log.LogInformation("Getting Immutable grid");
immutableTRexGrid = DIContext.Obtain <ITRexGridFactory>()?.Grid(StorageMutability.Immutable);
Log.LogInformation($"Got {immutableTRexGrid?.Name}");
// If there was no connection obtained, attempt to create a new instance
if (immutableTRexGrid == null)
{
var roleNames = roles.Aggregate("|", (s1, s2) => s1 + s2 + "|");
TRexNodeID = Guid.NewGuid();
//Log.LogInformation($"Creating new Ignite node with Roles = {roleNames} & TRexNodeId = {TRexNodeID}");
var cfg = new IgniteConfiguration
{
IgniteInstanceName = TRexGrids.ImmutableGridName(),
ClientMode = true,
JvmOptions = new List <string>()
{
"-DIGNITE_QUIET=false",
"-Djava.net.preferIPv4Stack=true",
"-XX:+UseG1GC",
"--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED",
"--add-exports=java.base/sun.nio.ch=ALL-UNNAMED",
"--add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED",
"--add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED",
"--add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED",
"--illegal-access=permit"
},
JvmMaxMemoryMb = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_JVM_MAX_HEAP_SIZE_MB, DEFAULT_IGNITE_JVM_MAX_HEAP_SIZE_MB),
JvmInitialMemoryMb = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_JVM_INITIAL_HEAP_SIZE_MB, DEFAULT_IGNITE_JVM_INITIAL_HEAP_SIZE_MB),
UserAttributes = new Dictionary <string, object>
{
{ "TRexNodeId", TRexNodeID.ToString() }
},
Logger = new TRexIgniteLogger(DIContext.Obtain <IConfigurationStore>(), Logger.CreateLogger("ImmutableClientServer")),
// Set an Ignite metrics heartbeat of 10 seconds
MetricsLogFrequency = new TimeSpan(0, 0, 0, 10),
PublicThreadPoolSize = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_PUBLIC_THREAD_POOL_SIZE, DEFAULT_IGNITE_PUBLIC_THREAD_POOL_SIZE),
SystemThreadPoolSize = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_SYSTEM_THREAD_POOL_SIZE, DEFAULT_IGNITE_SYSTEM_THREAD_POOL_SIZE),
PeerAssemblyLoadingMode = PeerAssemblyLoadingMode.CurrentAppDomain,
BinaryConfiguration = new BinaryConfiguration
{
Serializer = new BinarizableSerializer()
},
// Add the TRex progressive request custom thread pool
ExecutorConfiguration = new List <ExecutorConfiguration>
{
new ExecutorConfiguration
{
Name = BaseIgniteClass.TREX_PROGRESSIVE_QUERY_CUSTOM_THREAD_POOL_NAME,
Size = DIContext.Obtain <IConfigurationStore>().GetValueInt(PROGRESSIVE_REQUEST_CUSTOM_POOL_SIZE, DEFAULT_PROGRESSIVE_REQUEST_CUSTOM_POOL_SIZE)
}
}
};
foreach (var roleName in roles)
{
cfg.UserAttributes.Add($"{ServerRoles.ROLE_ATTRIBUTE_NAME}-{roleName}", "True");
}
bool.TryParse(Environment.GetEnvironmentVariable("IS_KUBERNETES"), out var isKubernetes);
cfg = isKubernetes ? setKubernetesIgniteConfiguration(cfg) : setLocalIgniteConfiguration(cfg);
try
{
immutableTRexGrid = DIContext.Obtain <ITRexGridFactory>()?.Grid(TRexGrids.ImmutableGridName(), cfg);
}
catch (Exception e)
{
Log.LogError(e, $"Creation of new Ignite node with Role = {roleNames} & TRexNodeId = {TRexNodeID} failed with Exception:");
throw;
}
finally
{
Log.LogInformation($"Completed creation of new Ignite node with Role = {roleNames} & TRexNodeId = {TRexNodeID}");
}
}
}
}
/// <summary>
/// Executes the life cycle of the service until it is aborted
/// </summary>
public void Execute(IServiceContext context)
{
try
{
if (_log == null)
{
Console.WriteLine($"Error: Null logger present in {nameof(SiteModelChangeProcessorService)}.{nameof(Execute)}");
}
_log.LogInformation($"{context.Name} starting executing");
Aborted = false;
_waitHandle = new EventWaitHandle(false, EventResetMode.AutoReset);
// Get the ignite grid and cache references
var ignite = DIContext.Obtain <ITRexGridFactory>()?.Grid(StorageMutability.Immutable) ??
Ignition.GetIgnite(TRexGrids.ImmutableGridName());
if (ignite == null)
{
_log.LogError("Ignite reference in service is null - aborting service execution");
return;
}
// Don't start operations until the local (immutable) grid is confirmed as active
DIContext.ObtainRequired <IActivatePersistentGridServer>().WaitUntilGridActive(TRexGrids.ImmutableGridName());
// Once active, delay start of operations for a time to ensure everything is up and running
var delay = DIContext.ObtainRequired <IConfigurationStore>().GetValueInt("TREX_SITE_MODEL_CHANGE_MAP_SERVICE_OPERATION_START_DELAY_SECONDS", 120);
_log.LogInformation($"Delaying start of operations for {delay} seconds");
Thread.Sleep(delay * 1000);
_log.LogInformation("Obtaining queue cache reference");
var queueCache = ignite.GetCache <ISiteModelChangeBufferQueueKey, ISiteModelChangeBufferQueueItem>(TRexCaches.SiteModelChangeBufferQueueCacheName());
_log.LogInformation($"Obtained queue cache for SiteModelChangeBufferQueueKey: {queueCache}");
var handler = new SiteModelChangeProcessorItemHandler();
var listener = new LocalSiteModelChangeListener(handler);
// Obtain the query handle for the continuous query from the DI context, or if not available create it directly
// Construct the continuous query machinery
// Set the initial query to return all elements in the cache
// Instantiate the queryHandle and start the continuous query on the remote nodes
// Note: Only cache items held on this local node will be handled here
var queryHandleFactory = DIContext.Obtain <Func <LocalSiteModelChangeListener, IContinuousQueryHandle <ICacheEntry <ISiteModelChangeBufferQueueKey, ISiteModelChangeBufferQueueItem> > > >();
if (queryHandleFactory != null)
{
_log.LogInformation("Obtaining query handle from DI factory");
_queryHandle = queryHandleFactory(listener);
}
while (_queryHandle == null && !Aborted)
{
_log.LogInformation("Obtaining query handle from QueryContinuous() API");
try
{
_queryHandle = queueCache.QueryContinuous
(qry: new ContinuousQuery <ISiteModelChangeBufferQueueKey, ISiteModelChangeBufferQueueItem>(listener)
{
Local = true
},
initialQry: new ScanQuery <ISiteModelChangeBufferQueueKey, ISiteModelChangeBufferQueueItem> {
Local = true
});
}
catch (Exception e)
{
_log.LogError(e, "Exception while constructing continuous query, will sleep and retry");
Thread.Sleep(5000);
}
}
if (_queryHandle == null || Aborted)
{
_log.LogInformation("No query handle available, or aborting");
return;
}
using (_queryHandle)
{
_log.LogInformation("Performing initial continuous query cursor scan of items to process");
// Perform the initial query to grab all existing elements and process them. Make sure to sort them in time order first
_queryHandle.GetInitialQueryCursor().OrderBy(x => x.Key.InsertUTCTicks).ForEach(handler.Add);
while (!Aborted)
{
try
{
{
// Cycle looking for new work to do as items arrive until aborted...
_log.LogInformation("Entering steady state continuous query scan of items to process");
// Activate the handler with the inject initial continuous query and move into steady state processing
InSteadyState = true;
handler.Activate();
do
{
_waitHandle.WaitOne(_serviceCheckIntervalMs);
} while (!Aborted);
}
}
catch (Exception e)
{
_log.LogError(e, "Site model change processor service unhandled exception, waiting and trying again");
// Sleep for 5 seconds to see if things come right and then try again
Thread.Sleep(5000);
}
}
}
handler.Cancel();
}
catch (Exception e)
{
_log.LogError(e, "Exception occurred performing initial set up of continuous query and scan of existing items");
}
finally
{
_log.LogInformation($"{context.Name} completed executing");
}
}
/// <summary>
/// Default no-arg constructor that sets up cluster and compute projections available for use
/// </summary>
public DesignProfilerServicePoolRequest() : base(TRexGrids.ImmutableGridName(), ServerRoles.DESIGN_PROFILER)
{
}
public void StopListening()
{
var listener = new SiteModelAttributesChangedEventListener(TRexGrids.ImmutableGridName());
listener.StopListening();
}
public CellDatumRequest_ApplicationService() : base(TRexGrids.ImmutableGridName(), ServerRoles.ASNODE)
{
}
public override void ConfigureTRexGrid(IgniteConfiguration cfg)
{
base.ConfigureTRexGrid(cfg);
cfg.IgniteInstanceName = TRexGrids.ImmutableGridName();
cfg.JvmOptions = new List <string>()
{
"-DIGNITE_QUIET=false",
"-Djava.net.preferIPv4Stack=true",
"-XX:+UseG1GC",
"--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED",
"--add-exports=java.base/sun.nio.ch=ALL-UNNAMED",
"--add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED",
"--add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED",
"--add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED",
"--illegal-access=permit"
};
cfg.JvmMaxMemoryMb = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_JVM_MAX_HEAP_SIZE_MB, DEFAULT_IGNITE_JVM_MAX_HEAP_SIZE_MB);
cfg.JvmInitialMemoryMb = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_JVM_INITIAL_HEAP_SIZE_MB, DEFAULT_IGNITE_JVM_INITIAL_HEAP_SIZE_MB);
cfg.UserAttributes = new Dictionary <string, object>
{
{ "Owner", TRexGrids.ImmutableGridName() },
};
// Configure the Ignite persistence layer to store our data
cfg.DataStorageConfiguration = new DataStorageConfiguration
{
WalMode = WalMode.Fsync,
PageSize = DataRegions.DEFAULT_IMMUTABLE_DATA_REGION_PAGE_SIZE,
StoragePath = Path.Combine(TRexServerConfig.PersistentCacheStoreLocation, "Immutable", "Persistence"),
WalPath = Path.Combine(TRexServerConfig.PersistentCacheStoreLocation, "Immutable", "WalStore"),
WalArchivePath = Path.Combine(TRexServerConfig.PersistentCacheStoreLocation, "Immutable", "WalArchive"),
WalSegmentSize = 512 * 1024 * 1024, // Set the WalSegmentSize to 512Mb to better support high write loads (can be set to max 2Gb)
MaxWalArchiveSize = (long)10 * 512 * 1024 * 1024, // Ensure there are 10 segments in the WAL archive at the defined segment size
DefaultDataRegionConfiguration = new DataRegionConfiguration
{
Name = DataRegions.DEFAULT_IMMUTABLE_DATA_REGION_NAME,
InitialSize = DIContext.Obtain <IConfigurationStore>().GetValueLong(
IMMUTABLE_DATA_REGION_INITIAL_SIZE_MB,
DEFAULT_IMMUTABLE_DATA_REGION_INITIAL_SIZE_MB) * 1024 * 1024,
MaxSize = DIContext.Obtain <IConfigurationStore>().GetValueLong(
IMMUTABLE_DATA_REGION_MAX_SIZE_MB,
DEFAULT_IMMUTABLE_DATA_REGION_MAX_SIZE_MB) * 1024 * 1024,
PersistenceEnabled = true
}
};
Log.LogInformation($"cfg.DataStorageConfiguration.StoragePath={cfg.DataStorageConfiguration.StoragePath}");
Log.LogInformation($"cfg.DataStorageConfiguration.WalArchivePath={cfg.DataStorageConfiguration.WalArchivePath}");
Log.LogInformation($"cfg.DataStorageConfiguration.WalPath={cfg.DataStorageConfiguration.WalPath}");
bool.TryParse(Environment.GetEnvironmentVariable("IS_KUBERNETES"), out var isKubernetes);
cfg = isKubernetes ? setKubernetesIgniteConfiguration(cfg) : setLocalIgniteConfiguration(cfg);
cfg.WorkDirectory = Path.Combine(TRexServerConfig.PersistentCacheStoreLocation, "Immutable");
cfg.Logger = new TRexIgniteLogger(DIContext.Obtain <IConfigurationStore>(), Logger.CreateLogger("ImmutableCacheComputeServer"));
// Set an Ignite metrics heartbeat of 10 seconds
cfg.MetricsLogFrequency = new TimeSpan(0, 0, 0, 10); // TODO: This needs to be added to configuration
cfg.PublicThreadPoolSize = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_PUBLIC_THREAD_POOL_SIZE, DEFAULT_IGNITE_PUBLIC_THREAD_POOL_SIZE);
cfg.SystemThreadPoolSize = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_SYSTEM_THREAD_POOL_SIZE, DEFAULT_IGNITE_SYSTEM_THREAD_POOL_SIZE);
cfg.PeerAssemblyLoadingMode = PeerAssemblyLoadingMode.CurrentAppDomain;
cfg.BinaryConfiguration = new BinaryConfiguration
{
Serializer = new BinarizableSerializer()
};
}
public ProfileRequest_ApplicationService() : base(TRexGrids.ImmutableGridName(), ServerRoles.ASNODE_PROFILER)
{
}
public GriddedReportRequest() : base(TRexGrids.ImmutableGridName(), ServerRoles.REPORTING_ROLE)
{
}
/// <summary>
/// Default no-arg constructor that sets up cluster and compute projections available for use
/// </summary>
public ApplicationServicePoolRequest() : base(TRexGrids.ImmutableGridName(), ServerRoles.ASNODE)
{
}
public QuantizedMeshRequest() : base(TRexGrids.ImmutableGridName(), ServerRoles.QNANTIZED_MESH_NODE)
{
}
public TileRenderRequest() : base(TRexGrids.ImmutableGridName(), ServerRoles.TILE_RENDERING_NODE)
{
}
public TemperatureStatisticsRequest_ApplicationService() : base(TRexGrids.ImmutableGridName(), ServerRoles.ASNODE)
{
}
static void Main(string[] args)
{
EnsureAssemblyDependenciesAreLoaded();
DependencyInjection();
try
{
Log = Logger.CreateLogger <Program>();
Log.LogInformation("Activating Grids");
Log.LogInformation(
"About to call ActivatePersistentGridServer.Instance().SetGridActive() for Mutable TRex grid");
bool result2 = DIContext.Obtain <IActivatePersistentGridServer>().SetGridActive(TRexGrids.MutableGridName());
Log.LogInformation(
"About to call ActivatePersistentGridServer.Instance().SetGridActive() for Immutable TRex grid");
bool result1 = DIContext.Obtain <IActivatePersistentGridServer>().SetGridActive(TRexGrids.ImmutableGridName());
Log.LogInformation($"Immutable Grid Active: {result1}");
if (!result1)
{
Log.LogCritical("Immutable Grid failed to activate");
}
Log.LogInformation($"Mutable Grid Active: {result2}");
if (!result2)
{
Log.LogCritical("Mutable Grid failed to activate");
}
}
finally
{
DIContext.Obtain <ITRexGridFactory>()?.StopGrids();
}
}
