private bool TryFill(SubscriptionBuffer buffer)
{
if (!buffer.IsPolling && !buffer.IsPoisoned && buffer.NewEventsQueue.Count < queueMaxCount)
{
buffer.IsPolling = true;
try
{
if (!ThreadPool.UnsafeQueueUserWorkItem(new WaitCallback(_ =>
poller.PollSubscription(buffer.StreamType, buffer.Url, buffer.Token, buffer.CurrentBufferVersion)), null))
{
this.bus.Publish(new FatalErrorOcurred(new FatalErrorException("A work item could not be queued in Poller.TryFill()")));
}
}
catch (Exception ex)
{
this.bus.Publish(new FatalErrorOcurred(new FatalErrorException("A work item could not be queued in Poller.TryFill()", ex)));
throw;
}
return true;
}
return false;
}
public void Handle(AddNewSubscriptionOnTheFly message)
{
lock (this.lockObjectForOnTheFlySub)
{
if (this.repository.TryAddNewSubscriptionOnTheFly(message.StreamType, message.Url, message.Token))
{
if (this.onTheFlyBufferPool == null)
this.onTheFlyBufferPool = new ConcurrentBag<SubscriptionBuffer>();
else
if (this.onTheFlyBufferPool.Any(x => x.StreamType == message.StreamType))
return;
var sub = new SubscriptionBuffer(message.StreamType, message.Url, message.Token, 0, false);
this.mainPublisherRegistry.Register(new OcassionallyConnectedSource(sub.StreamType, this.microserviceName));
this.onTheFlyBufferPool.Add(sub);
if (this.onTheFlySubscriptionsDetected)
return;
this.onTheFlySubscriptionsDetected = true;
if (this.onTheFlyThread != null)
throw new InvalidOperationException($"Already a thread of on the fly subscription running in poller of {this.microserviceName}.");
this.onTheFlyThread = new Thread(() =>
{
// This algorithm performs very well!
bool working;
while (!this.stopping)
{
working = false;
foreach (var buffer in this.onTheFlyBufferPool)
{
if (working)
{
this.TryFill(buffer);
this.TryFlush(buffer);
}
else
{
if (this.TryFlush(buffer) | this.TryFill(buffer))
{
working = true;
}
}
}
if (!working)
Thread.Sleep(1);
}
})
{
IsBackground = true,
Name = this.microserviceName + "_POLLER_ON-THE-FLY-SUBSCRIPTION"
};
this.onTheFlyThread.Start();
}
else
return;
}
}
private bool TryFlush(SubscriptionBuffer buffer)
{
//var eventsInQueueCount = buffer.NewEventsQueue.Count();
//if (eventsInQueueCount < 1 || buffer.EventsInProcessorByEcv.Any(e => !e.Value.WasProcessed))
//The buffer is empty or there are still events in the processor
//return false;
if (buffer.NewEventsQueue.IsEmpty)
return false;
long processorBufferVersion;
var eventsToProcess = new List<IEvent>();
if (buffer.EventsInProcessorByEcv.Any(e => !e.Value.WasProcessed))
{
while (!buffer.EventsInProcessorByEcv.IsEmpty)
{
EventInProcessorBucket toBeRemovedIfApplicable;
var min = buffer.EventsInProcessorByEcv.Min(x => x.Key);
if (buffer.EventsInProcessorByEcv[min].WasProcessed)
buffer.EventsInProcessorByEcv.TryRemove(min, out toBeRemovedIfApplicable);
else
break;
}
if (buffer.EventsInProcessorByEcv.IsEmpty)
return false;
processorBufferVersion = buffer.EventsInProcessorByEcv.Min(x => x.Key);
// there are still some events in the 'bag', try to add a few more
int freeSlots;
var eventsInProcessor = buffer.EventsInProcessorByEcv.Where(x => !x.Value.WasProcessed).Count();
if (eventsToFlushMaxCount > eventsInProcessor)
freeSlots = eventsToFlushMaxCount - eventsInProcessor;
else
// there could be events to process, but there are not empty slots...
return false;
for (int i = 0; i < freeSlots; i++)
{
IEvent @event;
if (buffer.NewEventsQueue.TryPeek(out @event))
{
if (!buffer.EventsInProcessorByEcv.Any(x => x.Value.Event.StreamId == @event.StreamId && !x.Value.WasProcessed))
{
// there is not an event of the same stream that is still in the processor
buffer.NewEventsQueue.TryDequeue(out @event);
eventsToProcess.Add(@event);
}
}
else
break;
}
if (eventsToProcess.Count < 1)
// no events could be try to process. No op
return false;
#if DEBUG
this.log.Trace($"Optimizing event processing for {eventsToProcess.Count} event/s from {buffer.ProducerName}");
#endif
}
else
{
// if all events where processed, go here
for (int i = 0; i < eventsToFlushMaxCount; i++)
{
IEvent @event;
if (!buffer.NewEventsQueue.TryDequeue(out @event))
break;
eventsToProcess.Add(@event);
}
// the minimun of the queue
processorBufferVersion = eventsToProcess.Min(e => e.EventCollectionVersion);
// Reset the bag;
buffer.EventsInProcessorByEcv.Clear();
}
var streams = eventsToProcess.Select(incomingEvent =>
{
var message = (Message)incomingEvent;
message.ProcessorBufferVersion = processorBufferVersion;
message.StreamType = buffer.StreamType;
buffer.EventsInProcessorByEcv.TryAdd(incomingEvent.EventCollectionVersion, new EventInProcessorBucket(incomingEvent));
return incomingEvent;
//this.bus.Publish(new NewIncomingEvents(incomingEvent));
});
this.bus.Publish(new NewIncomingEvents(streams));
// Here we persist the current subscription version state
this.repository.PersistSubscriptionVersion(buffer.StreamType, processorBufferVersion, buffer.ProducerVersion);
return true;
}
private void RegisterOcassionallyConnectedSourceIfApplicable(SubscriptionBuffer subscription)
{
if (subscription.Token == Constants.OcassionallyConnectedSourceToken)
this.mainPublisherRegistry.Register(new OcassionallyConnectedSource(subscription.StreamType, this.microserviceName));
}
private bool TryFlush(SubscriptionBuffer buffer)
{
long processorBufferVersion;
var eventsInQueueCount = buffer.NewEventsQueue.Count();
//if (eventsInQueueCount < 1 || buffer.EventsInProcessorByEcv.Any(e => !e.Value.WasProcessed))
//The buffer is empty or there are still events in the processor
//return false;
if (eventsInQueueCount < 1)
return false;
var eventsToProcess = new List<IEvent>();
if (buffer.EventsInProcessorByEcv.Any(e => !e.Value.WasProcessed))
{
while (!buffer.EventsInProcessorByEcv.IsEmpty)
{
EventInProcessorBucket toBeRemovedIfApplicable;
var min = buffer.EventsInProcessorByEcv.Min(x => x.Key);
if (buffer.EventsInProcessorByEcv[min].WasProcessed)
buffer.EventsInProcessorByEcv.TryRemove(min, out toBeRemovedIfApplicable);
else
break;
}
if (buffer.EventsInProcessorByEcv.IsEmpty)
return false;
processorBufferVersion = buffer.EventsInProcessorByEcv.Min(x => x.Key);
// there are still some events in the 'bag', try to add a few more
int freeSlots;
var eventsInProcessor = buffer.EventsInProcessorByEcv.Where(x => !x.Value.WasProcessed).Count();
if (eventsToFlushMaxCount > eventsInProcessor)
freeSlots = eventsToFlushMaxCount - eventsInProcessor;
else
// there could be events to process, but there are not empty slots...
return false;
for (int i = 0; i < freeSlots; i++)
{
IEvent @event;
if (buffer.NewEventsQueue.TryPeek(out @event))
{
if (!buffer.EventsInProcessorByEcv.Any(x => x.Value.Event.StreamId == @event.StreamId && !x.Value.WasProcessed))
{
// there is not an event of the same stream that is still in the processor
buffer.NewEventsQueue.TryDequeue(out @event);
eventsToProcess.Add(@event);
}
}
else
break;
}
if (eventsToProcess.Count < 1)
// no events could be try to process. No op
return false;
#if DEBUG
this.log.Trace($"Optimizing event processing for {eventsToProcess.Count} event/s from {buffer.ProducerName}");
#endif
}
else
{
// if all events where processed, go here
for (int i = 0; i < eventsToFlushMaxCount; i++)
{
IEvent @event;
if (!buffer.NewEventsQueue.TryDequeue(out @event))
break;
eventsToProcess.Add(@event);
}
// the minimun of the queue
processorBufferVersion = eventsToProcess.Min(e => e.EventCollectionVersion);
// Reset the bag;
buffer.EventsInProcessorByEcv.Clear();
}
var streams = eventsToProcess.Select(incomingEvent =>
{
((Message)incomingEvent).ProcessorBufferVersion = processorBufferVersion;
buffer.EventsInProcessorByEcv.TryAdd(incomingEvent.EventCollectionVersion, new EventInProcessorBucket(incomingEvent));
return incomingEvent;
//this.bus.Publish(new NewIncomingEvents(incomingEvent));
})
.GroupBy(x => x.StreamId, x => x,
(key, group) => new
{
StreamId = key,
Events = group
});
foreach (var stream in streams)
// no need to order by again
//this.bus.Publish(new NewIncomingEvents(stream.Events.OrderBy(x => x.Version)));
this.bus.Publish(new NewIncomingEvents(stream.Events));
this.log.Trace($"{this.microserviceName} is handling {eventsToProcess.Count} event/s of {buffer.StreamType} queue with {eventsInQueueCount} event/s pulled from {buffer.Url}");
return true;
}