private void AutoFlushUnacknowledgedBatchPointers(bool flushAnyway = false) { CheckIfPointerIsStoredAsAcknowledged(); lock (BatchUnacknowledgedConsumerMessagesToMerge) { if (flushAnyway == false) { // Flush unacknowledged message if (BatchUnacknowledgedConsumerMessagesToMerge.Count() >= _partitionConfiguration.SizeInMemory) { var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchUnacknowledgedConsumerMessagesToMerge.Values); ConsumerPointerContext.BulkInsertOrUpdate(BatchUnacknowledgedConsumerMessagesToMerge.Values.ToList()); RemoveRegisteredFromDictionary(BatchUnacknowledgedConsumerMessagesToMerge, batchToInsert); } } else { // Flush unacknowledged message if (BatchUnacknowledgedConsumerMessagesToMerge.Count() != 0) { var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchUnacknowledgedConsumerMessagesToMerge.Values); ConsumerPointerContext.BulkInsertOrUpdate(BatchUnacknowledgedConsumerMessagesToMerge.Values.ToList()); RemoveRegisteredFromDictionary(BatchUnacknowledgedConsumerMessagesToMerge, batchToInsert); } } } }
public ConsumerConnector(ILogger <ConsumerIOService> logger, string tenant, string product, string component, string topic, string consumer, ConsumerPointerContext consumerPointer, PartitionConfiguration partitionConfiguration, int agentCount) { _logger = logger; _tenant = tenant; _product = product; _component = component; _topic = topic; _consumer = consumer; _partitionConfiguration = partitionConfiguration; ConsumerPointerContext = null; ThreadingPool = new Model.Threading.ThreadPool(agentCount); MessagesBuffer = new ConcurrentQueue <Model.Entities.ConsumerMessage>(); BatchAcknowledgedConsumerMessagesToMerge = new ConcurrentDictionary <Guid, Model.Entities.ConsumerMessage>(); BatchUnacknowledgedConsumerMessagesToMerge = new ConcurrentDictionary <Guid, Model.Entities.ConsumerMessage>(); ConsumerPointerContext = consumerPointer; Count = 0; try { consumerPointer.ChangeTracker.AutoDetectChangesEnabled = false; consumerPointer.Database.EnsureCreated(); // database exists // create new instance of Backend ConsumerArchiveBackgroundService _consumerArchiveBackgroundService = new ConsumerArchiveBackgroundService(logger, tenant, product, component, topic, consumer, partitionConfiguration, consumerPointer); _consumerArchiveBackgroundService.StartService(); } catch (Exception) { } _flushPointerTimer = new Timer(); _flushPointerTimer.Interval = partitionConfiguration.FlushInterval; _flushPointerTimer.Elapsed += FlushPointerTimer_Elapsed; _flushPointerTimer.AutoReset = true; _flushPointerTimer.Start(); }
private void CheckPointerDbConnection(ConsumerPointerContext tenantContext, string consumerKey) { int counter = 0; while (counter != 10) { if (tenantContext.Database.CanConnect()) { _logger.LogInformation($"Pointer database for '{consumerKey.Replace("~", "/")}' is responding"); break; } Thread.Sleep(1000); counter++; _logger.LogError($"Pointer database for {consumerKey.Replace("~", "/")} is not responding, trying to connect {counter} of 10"); } }
public ConsumerArchiveBackgroundService(ILogger <ConsumerIOService> logger, string tenant, string product, string component, string topic, string consumer, PartitionConfiguration partitionConfiguration, ConsumerPointerContext consumerPointerContext) { _logger = logger; _tenant = tenant; _product = product; _component = component; _topic = topic; _consumer = consumer; _partitionConfiguration = partitionConfiguration; _consumerPointerContext = consumerPointerContext; InitializeBackgroundTask(); }
private void AutoFlushAcknowledgedBatchPointers() { lock (BatchAcknowledgedConsumerMessagesToMerge) { if (ThreadingPool.AreThreadsRunning == true) { if (BatchAcknowledgedConsumerMessagesToMerge.Count >= _partitionConfiguration.SizeInMemory) { var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchAcknowledgedConsumerMessagesToMerge.Values); ConsumerPointerContext.BulkInsertOrUpdate(batchToInsert); RemoveRegisteredFromDictionary(BatchAcknowledgedConsumerMessagesToMerge, batchToInsert); } } else { if (BatchAcknowledgedConsumerMessagesToMerge.Count != 0) { var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchAcknowledgedConsumerMessagesToMerge.Values); ConsumerPointerContext.BulkInsertOrUpdate(batchToInsert); RemoveRegisteredFromDictionary(BatchAcknowledgedConsumerMessagesToMerge, batchToInsert); } } } }
private async Task TransmitUnacknowledgedMessages(ConsumerConnectedArgs obj) { int timeoutCounter = 0; while (File.Exists(ConsumerLocations.GetConsumerPointerFile(obj.Tenant, obj.Product, obj.Component, obj.Topic, obj.ConsumerName)) != true) { // try every second for 5 seconds if the db is created. timeoutCounter++; Thread.Sleep(1000); if (timeoutCounter == 5) { return; } } string consumerKey = GenerateConsumerKey(obj.Tenant, obj.Product, obj.Component, obj.Topic, obj.ConsumerName); ConsumerPointerContext consumerPointerContext = _consumerIOService.GetConsumerConnector(consumerKey).ConsumerPointerContext; try { consumerPointerContext = _consumerIOService.GetConsumerConnector(consumerKey).ConsumerPointerContext; } catch (Exception) { _logger.LogError($"Couldn't sent unacknoledge messages to consumer '{obj.ConsumerName}' at {consumerKey.Replace("~", "/")}"); ReleaseUnacknoledgedMessageTasks(consumerKey); return; } List <MessageFile> partitionFiles = GetPartitionFiles(obj.Tenant, obj.Product, obj.Component, obj.Topic); bool isNewConsumer = false; try { // check if connection is open CheckPointerDbConnection(consumerPointerContext, consumerKey); var unackedMessages = consumerPointerContext.ConsumerMessages.Where(x => x.IsAcknowledged == false).OrderBy(x => x.SentDate).ToList(); if (unackedMessages.Count == 0) { int totalCount = consumerPointerContext.ConsumerMessages.Count(); if (totalCount == 0) { // Checking if this is a new consumer. if (obj.InitialPosition == InitialPosition.Latest) { return; } unackedMessages = consumerPointerContext.ConsumerMessages.OrderBy(x => x.SentDate).ToList(); isNewConsumer = true; } } await AnalysePartitionFiles(obj, partitionFiles, isNewConsumer, unackedMessages); } catch (Exception ex) { _logger.LogError($"Couldn't sent unacknoledge messages to consumer '{obj.ConsumerName}' at {consumerKey.Replace("~", "/")}; errorDetails = {ex.Message}"); } ReleaseUnacknoledgedMessageTasks(consumerKey); }