Exemple #1
0
        private bool ActivateMutable()
        {
            Log.LogInformation("About to call ActivatePersistentGridServer.Instance().SetGridActive() for Mutable TRex grid");
            bool result = ActivatePersistentGridServer.Instance().SetGridActive(TRexGrids.MutableGridName());

            Log.LogInformation($"Activation process completed: Mutable = {result}");

            return(result);
        }
Exemple #2
0
        /// <summary>
        /// Creates an appropriate new Ignite client node depending on the TRex Grid it is being attached to
        /// </summary>
        /// <param name="gridName"></param>
        /// <param name="role"></param>
        /// <returns></returns>
        public static IgniteServer NewClientNode(string gridName, string role)
        {
            if (gridName.Equals(TRexGrids.MutableGridName()))
            {
                return(new MutableClientServer(role));
            }

            if (gridName.Equals(TRexGrids.ImmutableGridName()))
            {
                return(new ImmutableClientServer(role));
            }

            throw new ArgumentException($"{gridName} is an unknown grid to create a client node within.");
        }
Exemple #3
0
        public void StartTRexGridCacheNode()
        {
            var cfg = new IgniteConfiguration();

            ConfigureTRexGrid(cfg);

            _log.LogInformation($"Creating new Ignite node for {cfg.IgniteInstanceName}");

            try
            {
                mutableTRexGrid = DIContext.Obtain <ITRexGridFactory>()?.Grid(TRexGrids.MutableGridName(), cfg);
            }
            finally
            {
                _log.LogInformation($"Completed creation of new Ignite node: Exists = {mutableTRexGrid != null}, Factory available = {DIContext.Obtain<ITRexGridFactory>() != null}");
            }

            // Wait until the grid is active
            DIContext.Obtain <IActivatePersistentGridServer>().WaitUntilGridActive(TRexGrids.MutableGridName());

            // Add the mutable Spatial & NonSpatial caches
            InstantiateNonSpatialCacheReference();

            InstantiateSpatialSubGridDirectoryCacheReference();
            InstantiateSpatialSubGridSegmentCacheReference();

            InstantiateTAGFileBufferQueueCacheReference();

            InstantiateSiteModelExistenceMapsCacheReference();
            InstantiateSiteModelsCacheReference();

            InstantiateRebuildSiteModelCacheReferences();

            InstantiateDesignTopologyExistenceMapsCache();

            // Create the SiteModel MetaData Manager so later DI context references wont need to create the cache etc for it at an inappropriate time
            var _ = DIContext.Obtain <ISiteModelMetadataManager>();
        }
Exemple #4
0
        /// <summary>
        /// Constructor that creates a new server instance with a set of roles
        /// </summary>
        /// <param name="roles"></param>
        public MutableClientServer(string[] roles)
        {
            if (mutableTRexGrid == null)
            {
                // Attempt to attach to an already existing Ignite instance
                _log.LogInformation("Getting mutable grid");
                mutableTRexGrid = DIContext.Obtain <ITRexGridFactory>()?.Grid(StorageMutability.Mutable);
                _log.LogInformation($"Got {mutableTRexGrid?.Name}");

                // If there was no connection obtained, attempt to create a new instance
                if (mutableTRexGrid == null)
                {
                    var roleNames = roles.Aggregate("|", (s1, s2) => s1 + s2 + "|");

                    TRexNodeID = Guid.NewGuid();

                    _log.LogInformation($"Creating new Ignite node with Roles = {roleNames} & TRexNodeId = {TRexNodeID}");

                    var cfg = new IgniteConfiguration()
                    {
                        IgniteInstanceName = TRexGrids.MutableGridName(),
                        ClientMode         = true,

                        JvmOptions = new List <string>()
                        {
                            "-DIGNITE_QUIET=false",
                            "-Djava.net.preferIPv4Stack=true",
                            "-XX:+UseG1GC",
                            "--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED",
                            "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED",
                            "--add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED",
                            "--add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED",
                            "--add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED",
                            "--illegal-access=permit"
                        },

                        JvmMaxMemoryMb     = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_JVM_MAX_HEAP_SIZE_MB, DEFAULT_IGNITE_JVM_MAX_HEAP_SIZE_MB),
                        JvmInitialMemoryMb = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_JVM_INITIAL_HEAP_SIZE_MB, DEFAULT_IGNITE_JVM_INITIAL_HEAP_SIZE_MB),

                        UserAttributes = new Dictionary <string, object>()
                        {
                            { "TRexNodeId", TRexNodeID.ToString() }
                        },

                        Logger = new TRexIgniteLogger(DIContext.Obtain <IConfigurationStore>(), Logger.CreateLogger("MutableClientServer")),

                        // Set an Ignite metrics heartbeat of 10 seconds
                        MetricsLogFrequency = new TimeSpan(0, 0, 0, 10),

                        PublicThreadPoolSize = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_PUBLIC_THREAD_POOL_SIZE, DEFAULT_IGNITE_PUBLIC_THREAD_POOL_SIZE),
                        SystemThreadPoolSize = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_SYSTEM_THREAD_POOL_SIZE, DEFAULT_IGNITE_SYSTEM_THREAD_POOL_SIZE),

                        PeerAssemblyLoadingMode = PeerAssemblyLoadingMode.CurrentAppDomain,

                        BinaryConfiguration = new BinaryConfiguration
                        {
                            Serializer = new BinarizableSerializer()
                        }
                    };

                    foreach (var roleName in roles)
                    {
                        cfg.UserAttributes.Add($"{ServerRoles.ROLE_ATTRIBUTE_NAME}-{roleName}", "True");
                    }

                    bool.TryParse(Environment.GetEnvironmentVariable("IS_KUBERNETES"), out var isKubernetes);
                    cfg = isKubernetes ? setKubernetesIgniteConfiguration(cfg) : setLocalIgniteConfiguration(cfg);

                    try
                    {
                        base.ConfigureTRexGrid(cfg);
                        mutableTRexGrid = DIContext.Obtain <ITRexGridFactory>()?.Grid(TRexGrids.MutableGridName(), cfg);
                    }
                    catch (Exception e)
                    {
                        _log.LogError(e, $"Creation of new Ignite node with Role = {roleNames} & TRexNodeId = {TRexNodeID} failed with Exception:");
                    }
                    finally
                    {
                        _log.LogInformation($"Completed creation of new Ignite node with Role = {roleNames} & TRexNodeId = {TRexNodeID}");
                    }
                }
            }
        }
Exemple #5
0
        /// <summary>
        /// No-arg constructor. Instantiates the continuous query and performs initial scan of elements that the remote filter
        /// will populate into the node-local groupers within the mutable grid.
        /// </summary>
        public SegmentRetirementQueueManager(bool runLocally)
        {
            _log.LogInformation("Establishing segment retirement queue cache context");

            // Get the ignite grid and cache references

            _ignite = DIContext.Obtain <ITRexGridFactory>()?.Grid(StorageMutability.Mutable) ?? Ignition.GetIgnite(TRexGrids.MutableGridName());
            var queueCache = _ignite.GetCache <ISegmentRetirementQueueKey, SegmentRetirementQueueItem>(TRexCaches.SegmentRetirementQueueCacheName());

            // Todo: Create a thread to periodically (needed if we don't go down the service route
            // ....

            _log.LogInformation("Completed segment retirement queue manager initialization");
        }
Exemple #6
0
        /// <summary>
        /// Executes the life cycle of the service until it is aborted
        /// </summary>
        public void Execute(IServiceContext context)
        {
            try
            {
                if (_log == null)
                {
                    Console.WriteLine($"Error: Null logger present in {nameof(TAGFileBufferQueueService)}.{nameof(Execute)}");
                }

                _log.LogInformation($"{nameof(TAGFileBufferQueueService)} {context.Name} starting executing");

                _aborted    = false;
                _waitHandle = new EventWaitHandle(false, EventResetMode.AutoReset);

                // Get the ignite grid and cache references

                var ignite = DIContext.Obtain <ITRexGridFactory>()?.Grid(StorageMutability.Mutable) ??
                             Ignition.GetIgnite(TRexGrids.MutableGridName());

                if (ignite == null)
                {
                    _log.LogError("Ignite reference in service is null - aborting service execution");
                    return;
                }

                // Don't start operations until the local (mutable) grid is confirmed as active
                DIContext.ObtainRequired <IActivatePersistentGridServer>().WaitUntilGridActive(TRexGrids.MutableGridName());

                // Once active, delay start of operations for a time to ensure everything is up and running
                var delay = DIContext.ObtainRequired <IConfigurationStore>().GetValueInt("TREX_TAG_FILE_BUFFER_QUEUE_SERVICE_OPERATION_START_DELAY_SECONDS", 120);
                _log.LogInformation($"Delaying start of operations for {delay} seconds");
                Thread.Sleep(delay * 1000);

                _log.LogInformation("Obtaining queue cache reference");
                var queueCache = ignite.GetCache <ITAGFileBufferQueueKey, TAGFileBufferQueueItem>(TRexCaches.TAGFileBufferQueueCacheName());

                _handler = new TAGFileBufferQueueItemHandler();

                while (_queryHandle == null && !_aborted)
                {
                    try
                    {
                        // Construct the continuous query machinery
                        // Set the initial query to return all elements in the cache
                        // Instantiate the queryHandle and start the continuous query on the remote nodes
                        // Note: Only cache items held on this local node will be handled here

                        _log.LogInformation("Obtaining continuous query handle");
                        _queryHandle = queueCache.QueryContinuous
                                           (qry: new ContinuousQuery <ITAGFileBufferQueueKey, TAGFileBufferQueueItem>(new LocalTAGFileListener(_handler))
                        {
                            Local = true
                        },
                                           initialQry: new ScanQuery <ITAGFileBufferQueueKey, TAGFileBufferQueueItem> {
                            Local = true
                        });
                    }
                    catch (Exception e)
                    {
                        _log.LogError(e, "Exception while constructing continuous query, will sleep and retry");
                        Thread.Sleep(5000);
                    }
                }

                if (_queryHandle == null || _aborted)
                {
                    _log.LogInformation("No query handle available, or aborting");
                    return;
                }

                using (_queryHandle)
                {
                    // Perform the initial query to grab all existing elements and add them to the grouper
                    _log.LogInformation("Performing initial continuous query cursor scan of items");
                    _queryHandle.GetInitialQueryCursor().ForEach(item => _handler.Add(item.Key));

                    // Transition into steady state looking for new elements in the cache via the continuous query
                    while (!_aborted)
                    {
                        try
                        {
                            // Cycle looking for new work to do as TAG files arrive until aborted...
                            _log.LogInformation("Entering steady state continuous query scan of items to process in TAGFileBufferQueue");

                            do
                            {
                                _waitHandle.WaitOne(_serviceCheckIntervalMs);
                                //Log.LogInformation("Continuous query scan of items to process in TAGFileBufferQueue still active");
                            } while (!_aborted);
                        }
                        catch (Exception e)
                        {
                            _log.LogError(e, "Tag file buffer service unhandled exception, waiting and trying again");

                            // Sleep for 5 seconds to see if things come right and then try again
                            Thread.Sleep(5000);
                        }
                    }
                }

                _handler.Cancel();
            }
            catch (Exception e)
            {
                _log.LogError(e, "Exception occurred performing initial set up of continuous query and scan of existing items");
            }
            finally
            {
                _log.LogInformation($"{nameof(TAGFileBufferQueueService)} {context.Name} completed executing");
            }
        }
Exemple #7
0
        static void Main(string[] args)
        {
            EnsureAssemblyDependenciesAreLoaded();
            DependencyInjection();

            try
            {
                Log = Logger.CreateLogger <Program>();

                Log.LogInformation("Activating Grids");

                Log.LogInformation(
                    "About to call ActivatePersistentGridServer.Instance().SetGridActive() for Mutable TRex grid");
                bool result2 = DIContext.Obtain <IActivatePersistentGridServer>().SetGridActive(TRexGrids.MutableGridName());

                Log.LogInformation(
                    "About to call ActivatePersistentGridServer.Instance().SetGridActive() for Immutable TRex grid");
                bool result1 = DIContext.Obtain <IActivatePersistentGridServer>().SetGridActive(TRexGrids.ImmutableGridName());

                Log.LogInformation($"Immutable Grid Active: {result1}");
                if (!result1)
                {
                    Log.LogCritical("Immutable Grid failed to activate");
                }

                Log.LogInformation($"Mutable Grid Active: {result2}");
                if (!result2)
                {
                    Log.LogCritical("Mutable Grid failed to activate");
                }
            }
            finally
            {
                DIContext.Obtain <ITRexGridFactory>()?.StopGrids();
            }
        }
Exemple #8
0
 public AddAlignmentRequest() : base(TRexGrids.MutableGridName(), ServerRoles.DATA_MUTATION_ROLE)
 {
 }
Exemple #9
0
        /// <summary>
        /// No-arg constructor. Instantiates the continuous query and performs initial scan of elements that the remote filter
        /// will populate into the node-local groupers within the mutable grid.
        /// </summary>
        public TAGFileBufferQueueManager(bool runLocally)
        {
            _log.LogInformation("Establishing Ignite and TAG file buffer queue cache contexts");

            // Get the ignite grid and cache references
            _ignite = DIContext.Obtain <ITRexGridFactory>()?.Grid(StorageMutability.Mutable) ?? Ignition.GetIgnite(TRexGrids.MutableGridName());
            var queueCache    = _ignite.GetCache <ITAGFileBufferQueueKey, TAGFileBufferQueueItem>(TRexCaches.TAGFileBufferQueueCacheName());
            var handler       = new TAGFileBufferQueueItemHandler();
            var tagFileFilter = new RemoteTAGFileFilter(handler);

            _log.LogInformation("Creating continuous query");

            // Construct the continuous query machinery
            // Set the initial query to return all elements in the cache
            // Instantiate the queryHandle and start the continuous query on the remote nodes
            // Note: Only cache items held on this local node will be handled here
            _queryHandle = queueCache.QueryContinuous
                               (qry: new ContinuousQuery <ITAGFileBufferQueueKey, TAGFileBufferQueueItem>(new LocalTAGFileListener(handler))
            {
                Local  = runLocally,
                Filter = tagFileFilter
            },
                               initialQry: new ScanQuery <ITAGFileBufferQueueKey, TAGFileBufferQueueItem>
            {
                Local  = runLocally,
                Filter = tagFileFilter
            });

            // Perform the initial query to grab all existing elements and add them to the grouper
            // All processing should happen on the remote node in the implementation of the TAGFileFilter remote filter
            foreach (var item in _queryHandle.GetInitialQueryCursor())
            {
                _log.LogError(
                    $"A cache entry ({item.Key}) from the TAG file buffer queue was passed back to the local scan query rather than intercepted by the remote filter");
            }

            _log.LogInformation("Completed TAG file buffer queue manager initialization");
        }
Exemple #10
0
 /// <summary>
 /// Default no-arg constructor that sets up cluster and compute projections available for use
 /// </summary>
 public CoordinateSystemRequest() : base(TRexGrids.MutableGridName(), ServerRoles.TAG_PROCESSING_NODE)
 {
 }
Exemple #11
0
        public override void ConfigureTRexGrid(IgniteConfiguration cfg)
        {
            base.ConfigureTRexGrid(cfg);

            cfg.IgniteInstanceName = TRexGrids.MutableGridName();

            cfg.JvmMaxMemoryMb     = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_JVM_MAX_HEAP_SIZE_MB, DEFAULT_IGNITE_JVM_MAX_HEAP_SIZE_MB);
            cfg.JvmInitialMemoryMb = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_JVM_INITIAL_HEAP_SIZE_MB, DEFAULT_IGNITE_JVM_INITIAL_HEAP_SIZE_MB);

            cfg.UserAttributes = new Dictionary <string, object>
            {
                { "Owner", TRexGrids.MutableGridName() }
            };

            // Configure the Ignite persistence layer to store our data
            cfg.DataStorageConfiguration = new DataStorageConfiguration
            {
                WalMode  = WalMode.Fsync,
                PageSize = DataRegions.DEFAULT_MUTABLE_DATA_REGION_PAGE_SIZE,

                StoragePath    = Path.Combine(TRexServerConfig.PersistentCacheStoreLocation, "Mutable", "Persistence"),
                WalPath        = Path.Combine(TRexServerConfig.PersistentCacheStoreLocation, "Mutable", "WalStore"),
                WalArchivePath = Path.Combine(TRexServerConfig.PersistentCacheStoreLocation, "Mutable", "WalArchive"),

                WalSegmentSize    = 512 * 1024 * 1024,            // Set the WalSegmentSize to 512Mb to better support high write loads (can be set to max 2Gb)
                MaxWalArchiveSize = (long)10 * 512 * 1024 * 1024, // Ensure there are 10 segments in the WAL archive at the defined segment size

                DefaultDataRegionConfiguration = new DataRegionConfiguration
                {
                    Name        = DataRegions.DEFAULT_MUTABLE_DATA_REGION_NAME,
                    InitialSize = DIContext.Obtain <IConfigurationStore>().GetValueLong(
                        MUTABLE_DATA_REGION_INITIAL_SIZE_MB,
                        DEFAULT_MUTABLE_DATA_REGION_INITIAL_SIZE_MB) * 1024 * 1024,
                    MaxSize = DIContext.Obtain <IConfigurationStore>().GetValueLong(
                        MUTABLE_DATA_REGION_MAX_SIZE_MB,
                        DEFAULT_MUTABLE_DATA_REGION_MAX_SIZE_MB) * 1024 * 1024,

                    PersistenceEnabled = true
                },

                // Establish a separate data region for the TAG file buffer queue
                DataRegionConfigurations = new List <DataRegionConfiguration>
                {
                    new DataRegionConfiguration
                    {
                        Name        = DataRegions.TAG_FILE_BUFFER_QUEUE_DATA_REGION,
                        InitialSize = DIContext.Obtain <IConfigurationStore>().GetValueLong(
                            TAG_FILE_BUFFER_QUEUE_DATA_REGION_INITIAL_SIZE_MB,
                            DEFAULT_TAG_FILE_BUFFER_QUEUE_DATA_REGION_INITIAL_SIZE_MB) * 1024 * 1024,
                        MaxSize = DIContext.Obtain <IConfigurationStore>().GetValueLong(
                            TAG_FILE_BUFFER_QUEUE_DATA_REGION_MAX_SIZE_MB,
                            DEFAULT_TAG_FILE_BUFFER_QUEUE_DATA_REGION_MAX_SIZE_MB) * 1024 * 1024,

                        PersistenceEnabled = true
                    }
                }
            };

            cfg.CacheConfiguration = new List <CacheConfiguration>();

            _log.LogInformation($"cfg.DataStorageConfiguration.StoragePath={cfg.DataStorageConfiguration.StoragePath}");
            _log.LogInformation($"cfg.DataStorageConfiguration.WalArchivePath={cfg.DataStorageConfiguration.WalArchivePath}");
            _log.LogInformation($"cfg.DataStorageConfiguration.WalPath={cfg.DataStorageConfiguration.WalPath}");

            cfg.JvmOptions = new List <string>()
            {
                "-DIGNITE_QUIET=false",
                "-Djava.net.preferIPv4Stack=true",
                "-XX:+UseG1GC",
                "--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED",
                "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED",
                "--add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED",
                "--add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED",
                "--add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED",
                "--illegal-access=permit"
            };


            cfg.Logger = new TRexIgniteLogger(DIContext.Obtain <IConfigurationStore>(), Logger.CreateLogger("MutableCacheComputeServer"));

            // Set an Ignite metrics heartbeat of 10 seconds
            cfg.MetricsLogFrequency = new TimeSpan(0, 0, 0, 10); // TODO: This needs to be added to configuration

            cfg.PublicThreadPoolSize = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_PUBLIC_THREAD_POOL_SIZE, DEFAULT_IGNITE_PUBLIC_THREAD_POOL_SIZE);
            cfg.SystemThreadPoolSize = DIContext.Obtain <IConfigurationStore>().GetValueInt(IGNITE_SYSTEM_THREAD_POOL_SIZE, DEFAULT_IGNITE_SYSTEM_THREAD_POOL_SIZE);

            cfg.PeerAssemblyLoadingMode = PeerAssemblyLoadingMode.CurrentAppDomain;

            cfg.BinaryConfiguration = new BinaryConfiguration
            {
                Serializer = new BinarizableSerializer()
            };

            bool.TryParse(Environment.GetEnvironmentVariable("IS_KUBERNETES"), out var isKubernetes);
            cfg = isKubernetes ? setKubernetesIgniteConfiguration(cfg) : setLocalIgniteConfiguration(cfg);
            cfg.WorkDirectory = Path.Combine(TRexServerConfig.PersistentCacheStoreLocation, "Mutable");
        }
Exemple #12
0
        public TestQueueHolder()
        {
            //  MutableClientServer Server = new MutableClientServer(new [] { "TestQueue2" });
            IIgnite Ignite = DIContext.Obtain <ITRexGridFactory>()?.Grid(StorageMutability.Mutable) ?? Ignition.GetIgnite(TRexGrids.MutableGridName());

            QueueCache = Ignite.GetOrCreateCache <long, TestQueueItem>(
                new CacheConfiguration
            {
                Name          = "TestQueueCache2",
                QueryEntities = new[] {
                    new QueryEntity(typeof(long), typeof(TestQueueItem))
                },
                KeepBinaryInStore = true
            });

            Add(DateTime.UtcNow, "First");
            Add(DateTime.UtcNow, "Second");
            Add(DateTime.UtcNow, "Third");
            Add(DateTime.UtcNow, "Fourth");
            Add(DateTime.UtcNow, "Fifth");
        }
Exemple #13
0
        /// <summary>
        /// Executes the life cycle of the service until it is aborted
        /// </summary>
        public void Execute(IServiceContext context)
        {
            try
            {
                _log.LogInformation($"{nameof(SegmentRetirementQueueService)} {context.Name} starting executing");

                _aborted    = false;
                _waitHandle = new EventWaitHandle(false, EventResetMode.AutoReset);

                // Get the ignite grid and cache references

                var mutableIgnite = DIContext.Obtain <ITRexGridFactory>()?.Grid(StorageMutability.Mutable) ?? Ignition.GetIgnite(TRexGrids.MutableGridName());

                if (mutableIgnite == null)
                {
                    _log.LogError("Mutable Ignite reference in service is null - aborting service execution");
                    return;
                }

                // Don't start operations until the local (mutable) grid is confirmed as active
                DIContext.ObtainRequired <IActivatePersistentGridServer>().WaitUntilGridActive(TRexGrids.MutableGridName());

                // Once active, delay start of operations for a time to ensure everything is up and running
                var delay = DIContext.ObtainRequired <IConfigurationStore>().GetValueInt("TREX_SEGMENT_RETIREMENT_QUEUE_SERVICE_OPERATION_START_DELAY_SECONDS", 120);
                _log.LogInformation($"Delaying start of operations for {delay} seconds");
                Thread.Sleep(delay * 1000);

                _log.LogInformation("Obtaining queue cache reference");

                var queueCache = mutableIgnite.GetCache <ISegmentRetirementQueueKey, SegmentRetirementQueueItem>(TRexCaches.TAGFileBufferQueueCacheName());

                var queue   = new SegmentRetirementQueue();
                var handler = new SegmentRetirementQueueItemHandler();

                // Cycle looking for new work to do until aborted...
                do
                {
                    try
                    {
                        // Obtain a specific local mutable storage proxy so as to have a local transactional proxy
                        // for this activity
                        var storageProxy = DIContext.Obtain <IStorageProxyFactory>().MutableGridStorage();

                        if (storageProxy.Mutability != StorageMutability.Mutable)
                        {
                            throw new TRexException("Non mutable storage proxy available to segment retirement queue");
                        }

                        _log.LogInformation("About to query retiree spatial streams from cache");

                        var earlierThan = DateTime.UtcNow - retirementAge;
                        // Retrieve the list of segments to be retired
                        var retirees = queue.Query(earlierThan);

                        // Pass the list to the handler for action
                        var retireesCount = retirees?.Count ?? 0;
                        if (retireesCount > 0)
                        {
                            _log.LogInformation($"About to retire {retireesCount} groups of spatial streams from mutable and immutable contexts");

                            if (handler.Process(storageProxy, queueCache, retirees))
                            {
                                if (_reportDetailedSegmentRetirementActivityToLog)
                                {
                                    _log.LogInformation($"Successfully retired {retireesCount} spatial streams from mutable and immutable contexts");
                                }

                                // Remove the elements from the segment retirement queue
                                queue.Remove(earlierThan);
                            }
                            else
                            {
                                _log.LogError($"Failed to retire {retireesCount} spatial streams from mutable and immutable contexts");
                            }
                        }
                    }
                    catch (Exception e)
                    {
                        _log.LogError(e, "Exception reported while obtaining new group of retirees to process:");
                    }

                    _waitHandle.WaitOne(SEGMENT_RETIREMENT_QUEUE_SERVICE_CHECK_INTERVAL_MS);
                } while (!_aborted);
            }
            catch (Exception e)
            {
                _log.LogError(e, $"Unhandled exception occurred in {nameof(SegmentRetirementQueueService)}");
            }
            finally
            {
                _log.LogInformation($"{nameof(SegmentRetirementQueueService)} {context.Name} completed executing");
            }
        }
Exemple #14
0
 /// <summary>
 /// Default no-arg constructor that sets up cluster and compute projections available for use by the TAG file processing pipeline
 /// </summary>
 public TAGFileProcessingPoolRequest() : base(TRexGrids.MutableGridName(), ServerRoles.TAG_PROCESSING_NODE)
 {
 }
Exemple #15
0
 public void ImmutableGridName()
 {
     TRexGrids.GridName(StorageMutability.Immutable).Should().NotBeNullOrWhiteSpace();
     TRexGrids.MutableGridName().Should().Be(TRexGrids.GridName(StorageMutability.Mutable));
 }
Exemple #16
0
        /// <summary>
        /// Creates or obtains a reference to an already created TAG file buffer queue
        /// </summary>
        private void InstantiateCache()
        {
            var ignite = DIContext.Obtain <ITRexGridFactory>()?.Grid(StorageMutability.Mutable) ?? Ignition.GetIgnite(TRexGrids.MutableGridName());

            _queueCache = ignite.GetCache <ITAGFileBufferQueueKey, TAGFileBufferQueueItem>(TRexCaches.TAGFileBufferQueueCacheName());

            if (_queueCache == null)
            {
                _log.LogInformation($"Failed to get Ignite cache {TRexCaches.TAGFileBufferQueueCacheName()}");
                throw new ArgumentException("Ignite cache not available");
            }
        }
Exemple #17
0
 /// <summary>
 /// Site model deletion requests are coordinated from the mutable grid. All project information in the mutable grid, and all projections
 /// of that data present in the immutable grid are removed.
 /// </summary>
 public DeleteSiteModelRequest() : base(TRexGrids.MutableGridName(), ServerRoles.TAG_PROCESSING_NODE)
 {
 }
Exemple #18
0
 public RemoveSurveyedSurfaceRequest() : base(TRexGrids.MutableGridName(), ServerRoles.DATA_MUTATION_ROLE)
 {
 }
Exemple #19
0
 public RemoveTTMDesignRequest() : base(TRexGrids.MutableGridName(), ServerRoles.DATA_MUTATION_ROLE)
 {
 }
Exemple #20
0
 /// <summary>
 /// Site model rebuild requests are coordinated from the mutable grid.
 /// </summary>
 public RebuildSiteModelRequest() : base(TRexGrids.MutableGridName(), ServerRoles.PROJECT_REBUILDER_ROLE)
 {
 }