private void AutoFlushUnacknowledgedBatchPointers(bool flushAnyway = false)
{
CheckIfPointerIsStoredAsAcknowledged();
lock (BatchUnacknowledgedConsumerMessagesToMerge)
{
if (flushAnyway == false)
{
// Flush unacknowledged message
if (BatchUnacknowledgedConsumerMessagesToMerge.Count() >= _partitionConfiguration.SizeInMemory)
{
var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchUnacknowledgedConsumerMessagesToMerge.Values);
ConsumerPointerContext.BulkInsertOrUpdate(BatchUnacknowledgedConsumerMessagesToMerge.Values.ToList());
RemoveRegisteredFromDictionary(BatchUnacknowledgedConsumerMessagesToMerge, batchToInsert);
}
}
else
{
// Flush unacknowledged message
if (BatchUnacknowledgedConsumerMessagesToMerge.Count() != 0)
{
var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchUnacknowledgedConsumerMessagesToMerge.Values);
ConsumerPointerContext.BulkInsertOrUpdate(BatchUnacknowledgedConsumerMessagesToMerge.Values.ToList());
RemoveRegisteredFromDictionary(BatchUnacknowledgedConsumerMessagesToMerge, batchToInsert);
}
}
}
}
public ConsumerConnector(ILogger <ConsumerIOService> logger,
string tenant,
string product,
string component,
string topic,
string consumer,
ConsumerPointerContext consumerPointer,
PartitionConfiguration partitionConfiguration,
int agentCount)
{
_logger = logger;
_tenant = tenant;
_product = product;
_component = component;
_topic = topic;
_consumer = consumer;
_partitionConfiguration = partitionConfiguration;
ConsumerPointerContext = null;
ThreadingPool = new Model.Threading.ThreadPool(agentCount);
MessagesBuffer = new ConcurrentQueue <Model.Entities.ConsumerMessage>();
BatchAcknowledgedConsumerMessagesToMerge = new ConcurrentDictionary <Guid, Model.Entities.ConsumerMessage>();
BatchUnacknowledgedConsumerMessagesToMerge = new ConcurrentDictionary <Guid, Model.Entities.ConsumerMessage>();
ConsumerPointerContext = consumerPointer;
Count = 0;
try
{
consumerPointer.ChangeTracker.AutoDetectChangesEnabled = false;
consumerPointer.Database.EnsureCreated();
// database exists
// create new instance of Backend ConsumerArchiveBackgroundService
_consumerArchiveBackgroundService = new ConsumerArchiveBackgroundService(logger, tenant, product, component, topic, consumer, partitionConfiguration, consumerPointer);
_consumerArchiveBackgroundService.StartService();
}
catch (Exception)
{
}
_flushPointerTimer = new Timer();
_flushPointerTimer.Interval = partitionConfiguration.FlushInterval;
_flushPointerTimer.Elapsed += FlushPointerTimer_Elapsed;
_flushPointerTimer.AutoReset = true;
_flushPointerTimer.Start();
}
private void CheckPointerDbConnection(ConsumerPointerContext tenantContext, string consumerKey)
{
int counter = 0;
while (counter != 10)
{
if (tenantContext.Database.CanConnect())
{
_logger.LogInformation($"Pointer database for '{consumerKey.Replace("~", "/")}' is responding");
break;
}
Thread.Sleep(1000);
counter++;
_logger.LogError($"Pointer database for {consumerKey.Replace("~", "/")} is not responding, trying to connect {counter} of 10");
}
}
public ConsumerArchiveBackgroundService(ILogger <ConsumerIOService> logger,
string tenant,
string product,
string component,
string topic,
string consumer,
PartitionConfiguration partitionConfiguration,
ConsumerPointerContext consumerPointerContext)
{
_logger = logger;
_tenant = tenant;
_product = product;
_component = component;
_topic = topic;
_consumer = consumer;
_partitionConfiguration = partitionConfiguration;
_consumerPointerContext = consumerPointerContext;
InitializeBackgroundTask();
}
private void AutoFlushAcknowledgedBatchPointers()
{
lock (BatchAcknowledgedConsumerMessagesToMerge)
{
if (ThreadingPool.AreThreadsRunning == true)
{
if (BatchAcknowledgedConsumerMessagesToMerge.Count >= _partitionConfiguration.SizeInMemory)
{
var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchAcknowledgedConsumerMessagesToMerge.Values);
ConsumerPointerContext.BulkInsertOrUpdate(batchToInsert);
RemoveRegisteredFromDictionary(BatchAcknowledgedConsumerMessagesToMerge, batchToInsert);
}
}
else
{
if (BatchAcknowledgedConsumerMessagesToMerge.Count != 0)
{
var batchToInsert = new List <Model.Entities.ConsumerMessage>(BatchAcknowledgedConsumerMessagesToMerge.Values);
ConsumerPointerContext.BulkInsertOrUpdate(batchToInsert);
RemoveRegisteredFromDictionary(BatchAcknowledgedConsumerMessagesToMerge, batchToInsert);
}
}
}
}
private async Task TransmitUnacknowledgedMessages(ConsumerConnectedArgs obj)
{
int timeoutCounter = 0;
while (File.Exists(ConsumerLocations.GetConsumerPointerFile(obj.Tenant, obj.Product, obj.Component, obj.Topic, obj.ConsumerName)) != true)
{
// try every second for 5 seconds if the db is created.
timeoutCounter++;
Thread.Sleep(1000);
if (timeoutCounter == 5)
{
return;
}
}
string consumerKey = GenerateConsumerKey(obj.Tenant, obj.Product, obj.Component, obj.Topic, obj.ConsumerName);
ConsumerPointerContext consumerPointerContext = _consumerIOService.GetConsumerConnector(consumerKey).ConsumerPointerContext;
try
{
consumerPointerContext = _consumerIOService.GetConsumerConnector(consumerKey).ConsumerPointerContext;
}
catch (Exception)
{
_logger.LogError($"Couldn't sent unacknoledge messages to consumer '{obj.ConsumerName}' at {consumerKey.Replace("~", "/")}");
ReleaseUnacknoledgedMessageTasks(consumerKey);
return;
}
List <MessageFile> partitionFiles = GetPartitionFiles(obj.Tenant, obj.Product, obj.Component, obj.Topic);
bool isNewConsumer = false;
try
{
// check if connection is open
CheckPointerDbConnection(consumerPointerContext, consumerKey);
var unackedMessages = consumerPointerContext.ConsumerMessages.Where(x => x.IsAcknowledged == false).OrderBy(x => x.SentDate).ToList();
if (unackedMessages.Count == 0)
{
int totalCount = consumerPointerContext.ConsumerMessages.Count();
if (totalCount == 0)
{
// Checking if this is a new consumer.
if (obj.InitialPosition == InitialPosition.Latest)
{
return;
}
unackedMessages = consumerPointerContext.ConsumerMessages.OrderBy(x => x.SentDate).ToList();
isNewConsumer = true;
}
}
await AnalysePartitionFiles(obj, partitionFiles, isNewConsumer, unackedMessages);
}
catch (Exception ex)
{
_logger.LogError($"Couldn't sent unacknoledge messages to consumer '{obj.ConsumerName}' at {consumerKey.Replace("~", "/")}; errorDetails = {ex.Message}");
}
ReleaseUnacknoledgedMessageTasks(consumerKey);
}