protected virtual List <SignalWrapper <SignalDispatch <TKey> > > Consolidate(
List <SignalWrapper <SignalDispatch <TKey> > > items)
{
List <SignalWrapper <SignalDispatch <TKey> > > newList = items
.Where(x => !x.Signal.ShouldBeConsolidated())
.ToList();
items.Where(x => x.Signal.ShouldBeConsolidated())
.GroupBy(x => new
{
x.Signal.ReceiverSubscriberId,
x.Signal.CategoryId,
x.Signal.SignalDispatchId
})
.Select(x =>
{
SignalWrapper <SignalDispatch <TKey> > signal = x.First();
signal.ConsolidatedSignals = x.ToArray();
return(signal);
})
.ToList()
.ForEach(newList.Add);
return(newList);
}
private void comboBox1_SelectedIndexChanged(object sender, EventArgs e)
{
SignalWrapper item = (SignalWrapper)comboBox1.SelectedItem;
m_activeSignal = item.Signal.TryGetSignal("Voltage Angle");
m_frameworkCtrl.Framework.SetAngleReference(m_activeSignal);
}
public override void ApplyResult(SignalWrapper <SignalEvent <TKey> > item
, ProcessingResult result)
{
if (result == ProcessingResult.Success)
{
_signalFlushJob.Delete(item);
}
else if (result == ProcessingResult.Fail)
{
item.Signal.FailedAttempts++;
item.IsUpdated = true;
_signalFlushJob.Return(item);
}
else if (result == ProcessingResult.Repeat)
{
//received only limited number of subscribers, need to repeat for next limited batch of subscribers
Append(item);
}
else if (result == ProcessingResult.NoHandlerFound)
{
_logger.LogError(SenderInternalMessages.EventQueue_HandlerNotFound, item.Signal.EventKey);
_signalFlushJob.Delete(item);
}
else if (result == ProcessingResult.ReturnToStorage)
{
_signalFlushJob.Return(item);
}
}
protected virtual void SplitEvent(SignalWrapper <SignalEvent <TKey> > item)
{
List <EventSettings <TKey> > eventSettings = _eventSettingsQueries
.SelectByKey(item.Signal.EventKey).Result;
if (eventSettings.Count == 0)
{
_eventQueue.ApplyResult(item, ProcessingResult.NoHandlerFound);
}
else if (eventSettings.Count == 1)
{
item.Signal.EventSettingsId = eventSettings.First().EventSettingsId;
item.IsUpdated = true;
ComposeAndApplyResult(item);
}
else if (eventSettings.Count > 1)
{
var splitedEvents = new List <SignalEvent <TKey> >();
foreach (EventSettings <TKey> settings in eventSettings)
{
SignalEvent <TKey> clone = item.Signal.CreateClone();
clone.SignalEventId = default(TKey);
clone.EventSettingsId = settings.EventSettingsId;
splitedEvents.Add(clone);
}
_eventQueue.ApplyResult(item, ProcessingResult.Success);
_eventQueue.Append(splitedEvents, false);
}
}
protected virtual IEnumerable <TemplateData[]> GetTemplateDataBatches(
SignalWrapper <SignalDispatch <TKey> > item, int?batchSize)
{
batchSize = batchSize ?? _defaultConsolidationBatchSize;
DateTime?previousBatchLatest = null;
var categories = new List <(int deliveryType, int category)>();
categories.Add((item.Signal.DeliveryType, item.Signal.CategoryId.Value));
while (true)
{
List <SignalDispatch <TKey> > sameCategoryDispatches = _signalDispatchQueries.SelectConsolidated(
pageSize: batchSize.Value,
subscriberIds: new List <TKey> {
item.Signal.ReceiverSubscriberId.Value
},
categories: categories,
createdBefore: item.Signal.SendDateUtc,
createdAfter: previousBatchLatest)
.Result;
if (sameCategoryDispatches.Count < batchSize)
{
break;
}
//assume already ordered by CreateDateUtc asc
previousBatchLatest = sameCategoryDispatches.LastOrDefault()?.CreateDateUtc;
yield return(sameCategoryDispatches
.Select(DeserializeTemplateData)
.Where(x => x != null)
.ToArray());
}
}
protected virtual void IncrementFailedAttempts(SignalWrapper <SignalDispatch <TKey> > item)
{
item.Signal.FailedAttempts++;
if (item.Signal.FailedAttempts >= MaxFailedAttempts)
{
_logger.LogError(SenderInternalMessages.DispatchQueue_MaxAttemptsReached,
MaxFailedAttempts, nameof(SignalDispatch <TKey>), item.Signal.SignalDispatchId);
}
}
public override void Return(SignalWrapper <SignalDispatch <TKey> > item)
{
base.Return(item);
if (item.IsConsolidationCompleted)
{
EnqueueItem(item, FlushAction.DeleteConsolidated);
}
}
protected virtual void ComposeAndApplyResult(SignalWrapper <SignalEvent <TKey> > item)
{
Stopwatch composeTimer = Stopwatch.StartNew();
EventHandleResult <SignalDispatch <TKey> > composeResult = ComposeDispatches(item);
composeTimer.Stop();
_eventQueue.ApplyResult(item, composeResult.Result);
_monitor.DispatchesComposed(item.Signal, composeTimer.Elapsed, composeResult.Result, composeResult.Items);
}
//methods
public virtual bool Execute(SignalWrapper <SignalDispatch <TKey> > item)
{
if (item.Signal.EventSettingsId == null)
{
//SignalDispatch can be provided by ISignalProver without EventSettingsId
return(true);
}
EventSettings <TKey> eventSettings = _eventSettingsQueries.Select(item.Signal.EventSettingsId.Value).Result;
if (eventSettings == null)
{
_logger.LogError(SenderInternalMessages.Common_NoServiceWithKeyFound,
typeof(EventSettings <TKey>), nameof(EventSettings <TKey> .EventSettingsId), item.Signal.EventSettingsId.Value);
_dispatchQueue.ApplyResult(item, ProcessingResult.NoHandlerFound);
return(false);
}
bool shouldConsolidate = ValidateRequiredProperties(item.Signal, eventSettings);
if (!shouldConsolidate)
{
//Continue processing dispatch without consolidation
return(true);
}
ITemplateDataConsolidator consolidator = MatchConsolidator(item.Signal, eventSettings.ConsolidatorId.Value);
if (consolidator == null)
{
_dispatchQueue.ApplyResult(item, ProcessingResult.NoHandlerFound);
return(false);
}
DispatchTemplate <TKey> dispatchTemplate = eventSettings.Templates == null
? null
: eventSettings.Templates.FirstOrDefault(x => EqualityComparer <TKey> .Default.Equals(x.DispatchTemplateId, item.Signal.DispatchTemplateId.Value));
if (dispatchTemplate == null)
{
_logger.LogError(SenderInternalMessages.Common_NoServiceWithKeyFound,
typeof(DispatchTemplate <TKey>), nameof(DispatchTemplate <TKey> .DispatchTemplateId), item.Signal.DispatchTemplateId.Value);
_dispatchQueue.ApplyResult(item, ProcessingResult.NoHandlerFound);
return(false);
}
IEnumerable <TemplateData[]> batches = GetTemplateDataBatches(item, consolidator.BatchSize);
TemplateData consolidatedData = consolidator.Consolidate(batches);
dispatchTemplate.Update(item.Signal, consolidatedData);
item.IsUpdated = true;
item.IsConsolidationCompleted = true;
return(true);
}
protected virtual void Unlock(SignalWrapper <SignalDispatch <TKey> > item)
{
if (item.Signal.LockedBy == null && item.Signal.LockedSinceUtc == null)
{
return;
}
item.Signal.LockedBy = null;
item.Signal.LockedSinceUtc = null;
item.IsUpdated = true;
}
private void ChkAllSignals_ItemCheck(object sender, ItemCheckEventArgs e)
{
SignalWrapper item = (SignalWrapper)ChkAllSignals.Items[e.Index];
if (e.NewValue == CheckState.Checked)
{
m_framework.ActivateSignalGroup(item.Signal);
}
else
{
m_framework.DeactivateSignalGroup(item.Signal);
}
}
//add to flush queue
public virtual void Delete(SignalWrapper <TSignal> item)
{
if (IsTemporaryStorageEnabled && item.TempStorageId != null)
{
_temporaryStorage.Delete(_temporaryStorageParameters, item.TempStorageId.Value);
item.TempStorageId = null;
}
if (item.IsPersistentlyStored)
{
_flushQueues[FlushAction.DeleteOne].Queue.Add(item);
}
}
protected virtual void Append(SignalWrapper <TSignal> item, int key)
{
AppendToTemporaryStorage(item);
lock (_queueLock)
{
if (!_itemsQueue.ContainsKey(key))
{
_itemsQueue.Add(key, new Queue <SignalWrapper <TSignal> >());
}
_itemsQueue[key].Enqueue(item);
}
}
//Flush results
public virtual void Flush()
{
lock (_queueLock)
{
foreach (KeyValuePair <int, Queue <SignalWrapper <TSignal> > > queue in _itemsQueue)
{
while (queue.Value.Count > 0)
{
SignalWrapper <TSignal> next = queue.Value.Dequeue();
ApplyResult(next, ProcessingResult.ReturnToStorage);
}
}
}
}
//processing methods
protected virtual void DequeueAll()
{
while (CanContinue())
{
SignalWrapper <SignalDispatch <TKey> > item = _dispatchQueue.DequeueNext();
if (item == null)
{
break;
}
StartNextTask(() => ProcessSignal(item));
}
WaitForCompletion();
}
public static void Apply <TStateEnum>(FSMWrapper fsmw)
where TStateEnum : IConvertible
{
//check states for signal wrapper action
var allStateNames = Enum.GetNames(typeof(TStateEnum));
foreach (var state in fsmw.targetFsm.States)
{
bool shouldContainSignal = false;
if (allStateNames.Contains(state.Name))
{
state.ColorIndex = (fsmw as IFSMStateDescriber).stateColour;
shouldContainSignal = true;
}
if (shouldContainSignal)
{
if (Array.Find(state.Actions, (a) => a is SignalWrapper) == null)
{
var newAction = new SignalWrapper(typeof(TStateEnum));
state.Actions = new FsmStateAction[] { newAction }.Concat(
state.Actions
).ToArray();
}
}
else
{
var i = Array.FindIndex(state.Actions,
(a) => (a is SignalWrapper && (a as SignalWrapper).enumType == typeof(TStateEnum).ToString()));
if (i >= 0)
{
var dest = new FsmStateAction[state.Actions.Length - 1];
if (i > 0)
{
Array.Copy(state.Actions, 0, dest, 0, i);
}
if (i < state.Actions.Length - 1)
{
Array.Copy(state.Actions, i + 1, dest, i, state.Actions.Length - i - 1);
}
state.Actions = dest;
}
}
state.SaveActions();
}
}
//methods
public bool Execute(SignalWrapper <SignalDispatch <TKey> > item)
{
ConsolidationLock <TKey> groupLock = _consolidationLockTracker.GetOrAddLock(item.Signal);
if (groupLock == null)
{
//other process cleared consolidation locks table in the middle
//just repeat processing later
_dispatchQueue.ApplyResult(item, ProcessingResult.Repeat);
return(false);
}
bool isLockedByCurrentInstance = groupLock.LockedBy == _settings.LockedByInstanceId;
if (!isLockedByCurrentInstance)
{
//this consolidation is handled by other Sender instance.
//databaes Signal will be removed after consolidation finished but responsible Sender instance.
return(false);
}
bool isConsolidationRoot = EqualityComparer <TKey> .Default.Equals(
groupLock.ConsolidationRootId, item.Signal.SignalDispatchId);
if (isConsolidationRoot)
{
//lock was created and this dispatch is now consolidation root.
//can start consolidation.
return(true);
}
bool willBeConsolidatedWithCurrentBatch = item.Signal.CreateDateUtc < groupLock.ConsolidationRootSendDateUtc;
if (willBeConsolidatedWithCurrentBatch)
{
//this dispatch will be atteched to ConsolidationRoot.
//databaes Signal will be removed after consolidation finished.
return(false);
}
//next consolidation batch should start after current batch is finished.
//because need to prevent selecting extra items from previous batch and attach them twice to different consolidation batches.
//repeat processing item later.
_dispatchQueue.ApplyResult(item, ProcessingResult.Repeat);
return(false);
}
protected virtual void AppendToTemporaryStorage(SignalWrapper <TSignal> item)
{
if (IsTemporaryStorageEnabled &&
item.IsPersistentlyStored == false &&
item.TempStorageId == null)
{
item.TempStorageId = Guid.NewGuid();
_temporaryStorage.Insert(_temporaryStorageParameters, item.TempStorageId.Value, item.Signal);
}
if (item.ConsolidatedSignals != null)
{
item.ConsolidatedSignals
.ToList()
.ForEach(AppendToTemporaryStorage);
}
}
//methods
public bool Execute(SignalWrapper <SignalDispatch <TKey> > item)
{
bool isLockingEnabled = _settings.IsDbLockStorageEnabled;
if (!isLockingEnabled)
{
return(true);
}
if (item.Signal.ShouldBeConsolidated())
{
//consolidated items have their own locks for the whole consolidated group of dispatches
return(true);
}
bool isLockExpired = _lockTracker.CheckNeedToExtendLock(item.Signal.SignalDispatchId);
if (!isLockExpired)
{
return(true);
}
//Lock again before sending
DateTime lockExpirationDate = _lockTracker.GetMaxExpiredLockSinceUtc();
bool lockSet = _dispatchQueries.SetLock(new List <TKey> {
item.Signal.SignalDispatchId
},
lockId: _settings.LockedByInstanceId.Value,
newLockSinceTimeUtc: DateTime.UtcNow,
existingLockSinceDateUtc: lockExpirationDate)
.Result;
if (lockSet)
{
return(true);
}
//If already expired and locked by another Sender instance, do nothing and let another instance to process.
_lockTracker.ForgetLocks(new List <TKey> {
item.Signal.SignalDispatchId
});
//cancel any future steps
return(false);
}
public virtual SignalWrapper <SignalEvent <TKey> > DequeueNext()
{
SignalWrapper <SignalEvent <TKey> > item = null;
lock (_queueLock)
{
foreach (KeyValuePair <int, Queue <SignalWrapper <SignalEvent <TKey> > > > group in _itemsQueue)
{
if (group.Value.Count > 0)
{
item = group.Value.Dequeue();
break;
}
}
}
return(item);
}
//methods
public virtual bool Execute(SignalWrapper <SignalDispatch <TKey> > item)
{
EventSettings <TKey> eventSettings = item.Signal.EventSettingsId == null
? null
: _eventSettingsQueries.Select(item.Signal.EventSettingsId.Value).Result;
if (eventSettings == null)
{
return(true);
}
if (eventSettings.StoreInHistory)
{
_flushQueues[FlushAction.Insert].Queue.Add(item.Signal);
}
return(true);
}
public virtual void RestoreFromTemporaryStorage()
{
if (!IsTemporaryStorageEnabled)
{
return;
}
Dictionary <Guid, TSignal> items = _temporaryStorage.Select(_temporaryStorageParameters);
foreach (KeyValuePair <Guid, TSignal> item in items)
{
var signal = new SignalWrapper <TSignal>(item.Value, false)
{
TempStorageId = item.Key
};
Append(signal);
}
}
//dequeue methods
public virtual SignalWrapper <SignalDispatch <TKey> > DequeueNext()
{
SignalWrapper <SignalDispatch <TKey> > item = null;
List <int> activeDeliveryTypes = _dispatcherRegistry.GetActiveDeliveryTypes(true);
lock (_queueLock)
{
foreach (KeyValuePair <int, Queue <SignalWrapper <SignalDispatch <TKey> > > > group in _itemsQueue)
{
if (activeDeliveryTypes.Contains(group.Key) && group.Value.Count > 0)
{
item = group.Value.Dequeue();
break;
}
}
}
return(item);
}
protected void ProcessSignal(SignalWrapper <SignalDispatch <TKey> > item)
{
try
{
foreach (IDispatchProcessingCommand <TKey> command in _processingCommands)
{
bool completed = command.Execute(item);
if (!completed)
{
break;
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, null);
_dispatchQueue.ApplyResult(item, ProcessingResult.Fail);
}
}
protected virtual async Task EnqueueSignalEvent(SignalEvent <TKey> signalEvent, SignalWriteConcern writeConcern)
{
writeConcern = _senderSettings.GetWriteConcernOrDefault(writeConcern);
bool ensurePersisted = writeConcern == SignalWriteConcern.PersistentStorage;
if (ensurePersisted)
{
await _eventQueries.Insert(new List <SignalEvent <TKey> > {
signalEvent
})
.ConfigureAwait(false);
}
var signalWrapper = new SignalWrapper <SignalEvent <TKey> >(signalEvent, ensurePersisted);
_eventQueue.Append(signalWrapper);
_monitor.EventReceived(signalEvent);
}
public virtual void Return(SignalWrapper <TSignal> item)
{
if (item.IsPersistentlyStored == false)
{
//Temp storage item will be deleted after flushing to permanent storage.
EnqueueItem(item, FlushAction.Insert);
}
if (item.IsPersistentlyStored == true && item.IsUpdated)
{
//Temp storage item will be deleted after flushing to permanent storage.
//Until then, preserve changes made to TempStorage.
if (IsTemporaryStorageEnabled && item.TempStorageId != null)
{
_temporaryStorage.Update(_temporaryStorageParameters, item.TempStorageId.Value, item.Signal);
}
EnqueueItem(item, FlushAction.Update);
}
}
protected void ProcessSignal(SignalWrapper <SignalEvent <TKey> > item)
{
try
{
if (item.Signal.EventSettingsId == null)
{
SplitEvent(item);
}
else
{
ComposeAndApplyResult(item);
}
}
catch (Exception ex)
{
_logger.LogError(ex, null);
//increment fail counter and don't let same event to repeat exceptions multiple times
_eventQueue.ApplyResult(item, ProcessingResult.Fail);
}
}
//apply result methods
public override void ApplyResult(SignalWrapper <SignalDispatch <TKey> > item, ProcessingResult result)
{
if (result == ProcessingResult.Success)
{
//successfuly sent dispatch
_signalFlushJob.Delete(item);
}
else if (result == ProcessingResult.Fail)
{
//dispatcher throws error
IncrementFailedAttempts(item);
Unlock(item);
item.Signal.SendDateUtc = DateTime.UtcNow.Add(RetryPeriod);
item.IsUpdated = true;
_signalFlushJob.Return(item);
}
else if (result == ProcessingResult.Repeat)
{
//dispatcher is not available
Unlock(item);
item.Signal.SendDateUtc = DateTime.UtcNow.Add(RetryPeriod);
item.IsUpdated = true;
_signalFlushJob.Return(item);
}
else if (result == ProcessingResult.NoHandlerFound)
{
//no dispatcher found matching deliveryType
IncrementFailedAttempts(item);
Unlock(item);
item.Signal.SendDateUtc = DateTime.UtcNow.Add(RetryPeriod);
item.IsUpdated = true;
_signalFlushJob.Return(item);
}
else if (result == ProcessingResult.ReturnToStorage)
{
//1. stoped Sender and saving everything from queue to database
//2. insert scheduled dispatch after it is composed
Unlock(item);
_signalFlushJob.Return(item);
}
}
//append methods
public virtual void Append(List <SignalDispatch <TKey> > signals, bool isPermanentlyStored)
{
SignalWrapper <SignalDispatch <TKey> >[] items = signals
.Select(x => SignalWrapper.Create(x, isPermanentlyStored))
.ToArray();
//return to database scheduled dispatches
SignalWrapper <SignalDispatch <TKey> >[] scheduledItems = items
.Where(x => x.Signal.IsScheduled && x.Signal.SendDateUtc > DateTime.UtcNow)
.ToArray();
foreach (SignalWrapper <SignalDispatch <TKey> > item in scheduledItems)
{
AppendToTemporaryStorage(item);
ApplyResult(item, ProcessingResult.ReturnToStorage);
}
//consolidate and enqueue immediate dispatches
List <SignalWrapper <SignalDispatch <TKey> > > immediateItems = items.Except(scheduledItems).ToList();
immediateItems = Consolidate(immediateItems);
immediateItems.ForEach(Append);
}
//methods
public virtual bool Execute(SignalWrapper <SignalDispatch <TKey> > item)
{
//find Dispatcher by deliveryType
IDispatchChannel <TKey> channel = _channelRegistry.Match(item.Signal);
if (channel == null)
{
_dispatchQueue.ApplyResult(item, ProcessingResult.NoHandlerFound);
return(false);
}
//send with dispatcher
ProcessingResult sendResult = ProcessingResult.Fail;
Stopwatch sendTimer = Stopwatch.StartNew();
try
{
sendResult = channel.Send(item.Signal);
}
catch (Exception ex)
{
_logger.LogError(ex, null);
}
sendTimer.Stop();
//check dispatcher availability
DispatcherAvailability availability = CheckAvailability(channel, sendResult);
if (sendResult == ProcessingResult.Fail && availability == DispatcherAvailability.NotAvailable)
{
sendResult = ProcessingResult.Repeat;
}
_monitor.DispatchSent(item.Signal, sendResult, sendTimer.Elapsed);
channel.CountSendAttempt(item.Signal, sendResult, availability);
_dispatchQueue.ApplyResult(item, sendResult);
return(sendResult == ProcessingResult.Success);
}
