private async Task AnalysePartitionFiles(ConsumerConnectedArgs obj, List <MessageFile> partitionFiles, bool isNewConsumer, IEnumerable <Model.Entities.ConsumerMessage> unackedMessages) { foreach (var paritionFile in partitionFiles) { // here we do check partitions db messages.... string topicName = _messageIOService.AddMessageFileConnectorGetKey(obj.Tenant, obj.Product, obj.Component, obj.Topic, paritionFile.PartitionDate); var partitionContext = _messageIOService.GetPartitionMessageContext(topicName, paritionFile.PartitionDate); // Get all rows var rows = partitionContext.Messages.OrderBy(x => x.SentDate).ToList(); // If is old consumer send only unacknowledged ones. if (isNewConsumer != true) { if (unackedMessages.Count() == 0) { return; } rows = partitionContext.Messages .Where(r => unackedMessages.Select(s => s.MessageId).Any(u => u == r.MessageId)) .OrderBy(x => x.SentDate) .ToList(); } await AnalyseFileRows(obj, rows); if (isNewConsumer != true) { unackedMessages.ToList().RemoveAll(r => rows.Any(u => u.MessageId == r.MessageId)); } } }
private async Task AnalyseFileRows(ConsumerConnectedArgs obj, List <Model.Entities.Message> rows) { foreach (var row in rows) { var consumerMessage = new ConsumerMessage() { Consumer = obj.ConsumerName, Message = new Message() { Tenant = obj.Tenant, Product = obj.Product, Component = obj.Component, Topic = obj.Topic, Id = row.MessageId, MessageRaw = row.Payload.JsonToObject <object>() } }; foreach (var xNode in _xNodeEventService.GetXNodeConnectionRepository().GetAllServices()) { //Transmit the message to the other nodes. await xNode.Value.Values.ToList()[0].GetHubConnection().SendAsync("TransmitMessagesToConsumer", consumerMessage); } } }
private void XNodeEventService_ConsumerUnacknowledgedMessagesRequested(ConsumerConnectedArgs obj) { string consumerKey = GenerateConsumerKey(obj.Tenant, obj.Product, obj.Component, obj.Topic, obj.ConsumerName); if (_unacknowledgedMessageProcesses.ContainsKey(consumerKey)) { return; } // We are adding the task to the Dictionary, when the task is done _unacknowledgedMessageProcesses.TryAdd(consumerKey, Task.Run(async() => await TransmitUnacknowledgedMessages(obj))); }
private void XNodeEventService_ConsumerConnected(ConsumerConnectedArgs obj) { _consumerIOService.TryCreateConsumerDirectory(obj.Tenant, obj.Product, obj.Component, obj.Topic, new Consumer() { Id = obj.Id, Name = obj.ConsumerName, Tenant = obj.Tenant, Product = obj.Product, Component = obj.Component, Topic = obj.Topic, SubscriptionType = obj.SubscriptionType, ConsumerSettings = new ConsumerSettings() { InitialPosition = obj.InitialPosition }, CreatedDate = DateTime.Now }); }
private async Task TransmitUnacknowledgedMessages(ConsumerConnectedArgs obj) { int timeoutCounter = 0; while (File.Exists(ConsumerLocations.GetConsumerPointerFile(obj.Tenant, obj.Product, obj.Component, obj.Topic, obj.ConsumerName)) != true) { // try every second for 5 seconds if the db is created. timeoutCounter++; Thread.Sleep(1000); if (timeoutCounter == 5) { return; } } string consumerKey = GenerateConsumerKey(obj.Tenant, obj.Product, obj.Component, obj.Topic, obj.ConsumerName); ConsumerPointerContext consumerPointerContext = _consumerIOService.GetConsumerConnector(consumerKey).ConsumerPointerContext; try { consumerPointerContext = _consumerIOService.GetConsumerConnector(consumerKey).ConsumerPointerContext; } catch (Exception) { _logger.LogError($"Couldn't sent unacknoledge messages to consumer '{obj.ConsumerName}' at {consumerKey.Replace("~", "/")}"); ReleaseUnacknoledgedMessageTasks(consumerKey); return; } List <MessageFile> partitionFiles = GetPartitionFiles(obj.Tenant, obj.Product, obj.Component, obj.Topic); bool isNewConsumer = false; try { // check if connection is open CheckPointerDbConnection(consumerPointerContext, consumerKey); var unackedMessages = consumerPointerContext.ConsumerMessages.Where(x => x.IsAcknowledged == false).OrderBy(x => x.SentDate).ToList(); if (unackedMessages.Count == 0) { int totalCount = consumerPointerContext.ConsumerMessages.Count(); if (totalCount == 0) { // Checking if this is a new consumer. if (obj.InitialPosition == InitialPosition.Latest) { return; } unackedMessages = consumerPointerContext.ConsumerMessages.OrderBy(x => x.SentDate).ToList(); isNewConsumer = true; } } await AnalysePartitionFiles(obj, partitionFiles, isNewConsumer, unackedMessages); } catch (Exception ex) { _logger.LogError($"Couldn't sent unacknoledge messages to consumer '{obj.ConsumerName}' at {consumerKey.Replace("~", "/")}; errorDetails = {ex.Message}"); } ReleaseUnacknoledgedMessageTasks(consumerKey); }