Ejemplo n.º 1
0
 private void AutoFlushUnacknowledgedBatchPointers(bool flushAnyway = false)
 {
     CheckIfPointerIsStoredAsAcknowledged();
     lock (BatchUnacknowledgedConsumerMessagesToMerge)
     {
         if (flushAnyway == false)
         {
             // Flush unacknowledged message
             if (BatchUnacknowledgedConsumerMessagesToMerge.Count() >= _partitionConfiguration.SizeInMemory)
             {
                 var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchUnacknowledgedConsumerMessagesToMerge.Values);
                 ConsumerPointerContext.BulkInsertOrUpdate(BatchUnacknowledgedConsumerMessagesToMerge.Values.ToList());
                 RemoveRegisteredFromDictionary(BatchUnacknowledgedConsumerMessagesToMerge, batchToInsert);
             }
         }
         else
         {
             // Flush unacknowledged message
             if (BatchUnacknowledgedConsumerMessagesToMerge.Count() != 0)
             {
                 var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchUnacknowledgedConsumerMessagesToMerge.Values);
                 ConsumerPointerContext.BulkInsertOrUpdate(BatchUnacknowledgedConsumerMessagesToMerge.Values.ToList());
                 RemoveRegisteredFromDictionary(BatchUnacknowledgedConsumerMessagesToMerge, batchToInsert);
             }
         }
     }
 }
Ejemplo n.º 2
0
        public ConsumerConnector(ILogger <ConsumerIOService> logger,
                                 string tenant,
                                 string product,
                                 string component,
                                 string topic,
                                 string consumer,
                                 ConsumerPointerContext consumerPointer,
                                 PartitionConfiguration partitionConfiguration,
                                 int agentCount)
        {
            _logger    = logger;
            _tenant    = tenant;
            _product   = product;
            _component = component;
            _topic     = topic;
            _consumer  = consumer;

            _partitionConfiguration = partitionConfiguration;


            ConsumerPointerContext = null;
            ThreadingPool          = new Model.Threading.ThreadPool(agentCount);

            MessagesBuffer = new ConcurrentQueue <Model.Entities.ConsumerMessage>();
            BatchAcknowledgedConsumerMessagesToMerge   = new ConcurrentDictionary <Guid, Model.Entities.ConsumerMessage>();
            BatchUnacknowledgedConsumerMessagesToMerge = new ConcurrentDictionary <Guid, Model.Entities.ConsumerMessage>();
            ConsumerPointerContext = consumerPointer;
            Count = 0;

            try
            {
                consumerPointer.ChangeTracker.AutoDetectChangesEnabled = false;
                consumerPointer.Database.EnsureCreated();

                // database exists
                // create new instance of Backend ConsumerArchiveBackgroundService
                _consumerArchiveBackgroundService = new ConsumerArchiveBackgroundService(logger, tenant, product, component, topic, consumer, partitionConfiguration, consumerPointer);
                _consumerArchiveBackgroundService.StartService();
            }
            catch (Exception)
            {
            }

            _flushPointerTimer           = new Timer();
            _flushPointerTimer.Interval  = partitionConfiguration.FlushInterval;
            _flushPointerTimer.Elapsed  += FlushPointerTimer_Elapsed;
            _flushPointerTimer.AutoReset = true;
            _flushPointerTimer.Start();
        }
Ejemplo n.º 3
0
        private void CheckPointerDbConnection(ConsumerPointerContext tenantContext, string consumerKey)
        {
            int counter = 0;

            while (counter != 10)
            {
                if (tenantContext.Database.CanConnect())
                {
                    _logger.LogInformation($"Pointer database for '{consumerKey.Replace("~", "/")}' is responding");
                    break;
                }
                Thread.Sleep(1000);
                counter++;
                _logger.LogError($"Pointer database for {consumerKey.Replace("~", "/")} is not responding, trying to connect {counter} of 10");
            }
        }
Ejemplo n.º 4
0
        public ConsumerArchiveBackgroundService(ILogger <ConsumerIOService> logger,
                                                string tenant,
                                                string product,
                                                string component,
                                                string topic,
                                                string consumer,
                                                PartitionConfiguration partitionConfiguration,
                                                ConsumerPointerContext consumerPointerContext)
        {
            _logger    = logger;
            _tenant    = tenant;
            _product   = product;
            _component = component;
            _topic     = topic;
            _consumer  = consumer;

            _partitionConfiguration = partitionConfiguration;
            _consumerPointerContext = consumerPointerContext;

            InitializeBackgroundTask();
        }
Ejemplo n.º 5
0
 private void AutoFlushAcknowledgedBatchPointers()
 {
     lock (BatchAcknowledgedConsumerMessagesToMerge)
     {
         if (ThreadingPool.AreThreadsRunning == true)
         {
             if (BatchAcknowledgedConsumerMessagesToMerge.Count >= _partitionConfiguration.SizeInMemory)
             {
                 var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchAcknowledgedConsumerMessagesToMerge.Values);
                 ConsumerPointerContext.BulkInsertOrUpdate(batchToInsert);
                 RemoveRegisteredFromDictionary(BatchAcknowledgedConsumerMessagesToMerge, batchToInsert);
             }
         }
         else
         {
             if (BatchAcknowledgedConsumerMessagesToMerge.Count != 0)
             {
                 var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchAcknowledgedConsumerMessagesToMerge.Values);
                 ConsumerPointerContext.BulkInsertOrUpdate(batchToInsert);
                 RemoveRegisteredFromDictionary(BatchAcknowledgedConsumerMessagesToMerge, batchToInsert);
             }
         }
     }
 }
Ejemplo n.º 6
0
        private async Task TransmitUnacknowledgedMessages(ConsumerConnectedArgs obj)
        {
            int timeoutCounter = 0;

            while (File.Exists(ConsumerLocations.GetConsumerPointerFile(obj.Tenant, obj.Product, obj.Component, obj.Topic, obj.ConsumerName)) != true)
            {
                // try every second for 5 seconds if the db is created.
                timeoutCounter++;
                Thread.Sleep(1000);
                if (timeoutCounter == 5)
                {
                    return;
                }
            }

            string consumerKey = GenerateConsumerKey(obj.Tenant, obj.Product, obj.Component, obj.Topic, obj.ConsumerName);
            ConsumerPointerContext consumerPointerContext = _consumerIOService.GetConsumerConnector(consumerKey).ConsumerPointerContext;

            try
            {
                consumerPointerContext = _consumerIOService.GetConsumerConnector(consumerKey).ConsumerPointerContext;
            }
            catch (Exception)
            {
                _logger.LogError($"Couldn't sent unacknoledge messages to consumer '{obj.ConsumerName}' at {consumerKey.Replace("~", "/")}");
                ReleaseUnacknoledgedMessageTasks(consumerKey);
                return;
            }

            List <MessageFile> partitionFiles = GetPartitionFiles(obj.Tenant, obj.Product, obj.Component, obj.Topic);
            bool isNewConsumer = false;

            try
            {
                // check if connection is open
                CheckPointerDbConnection(consumerPointerContext, consumerKey);

                var unackedMessages = consumerPointerContext.ConsumerMessages.Where(x => x.IsAcknowledged == false).OrderBy(x => x.SentDate).ToList();
                if (unackedMessages.Count == 0)
                {
                    int totalCount = consumerPointerContext.ConsumerMessages.Count();
                    if (totalCount == 0)
                    {
                        // Checking if this is a new consumer.
                        if (obj.InitialPosition == InitialPosition.Latest)
                        {
                            return;
                        }

                        unackedMessages = consumerPointerContext.ConsumerMessages.OrderBy(x => x.SentDate).ToList();
                        isNewConsumer   = true;
                    }
                }

                await AnalysePartitionFiles(obj, partitionFiles, isNewConsumer, unackedMessages);
            }
            catch (Exception ex)
            {
                _logger.LogError($"Couldn't sent unacknoledge messages to consumer '{obj.ConsumerName}' at {consumerKey.Replace("~", "/")}; errorDetails = {ex.Message}");
            }

            ReleaseUnacknoledgedMessageTasks(consumerKey);
        }