private async Task OnEventAppearead(IStreamSubscription arg1,
ResolvedEvent arg2,
IDictionary <string, Func <IStreamSubscription, ResolvedEvent, CancellationToken, Task> > methods,
IDictionary <string, Type> handlers,
IDictionary <string, Type> eventToTypeDict,
CancellationToken arg3)
{
var eventType = arg2.Event.EventType;
var func = methods.GetOrAdd(eventType, key =>
{
var p1 = Expression.Parameter(typeof(IStreamSubscription), "arg1");
var p2 = Expression.Parameter(typeof(ResolvedEvent), "arg2");
var p3 = Expression.Parameter(typeof(CancellationToken), "arg3");
var t = Expression.Constant(this);
var et = eventToTypeDict[key];
var h = handlers[key];
var c = Expression.Call(t, _onSpecMeth.MakeGenericMethod(et, h), p1, p2, p3);
var ret = Expression
.Lambda <Func <IStreamSubscription, ResolvedEvent, CancellationToken, Task> >(c, new[] { p1, p2, p3 })
.Compile();
return(ret);
});
await func(arg1, arg2, arg3);
}
public async Task StreamMessageReceived(IStreamSubscription subscription, StreamMessage streamMessage,
CancellationToken cancellationToken)
{
if (streamMessage.Position >= _lastCheckpoint)
{
var @event = await DeserializeJsonEvent(streamMessage, cancellationToken);
var transactions = new List <Transaction>()
{
new Transaction()
{
Id = subscription.Name,
Checkpoint = streamMessage.Position,
StreamId = subscription.StreamId,
Events = new List <EventEnvelope>()
{
new EventEnvelope()
{
Body = @event
}
}
}
};
await _subscriber.HandleTransactions(transactions, new SubscriptionInfo()
{
Id = "transaction",
CancellationToken = cancellationToken,
Subscription = this
});
}
}
public IDisposable WatchForChanges(int version, OnSettingsChanged onSettingsChanged, CancellationToken ct)
{
IStreamSubscription subscription = null;
async Task StreamMessageReceived(IStreamSubscription _, StreamMessage streamMessage,
CancellationToken cancellationToken)
{
var settings = await StreamStoreConfigRepository.BuildConfigurationSettingsFromMessage(streamMessage, _messageHooks, ct);
await onSettingsChanged(settings, ct);
};
void SubscriptionDropped(IStreamSubscription _, SubscriptionDroppedReason reason,
Exception exception = null)
{
if (reason != SubscriptionDroppedReason.Disposed)
{
SetupSubscription();
}
};
void SetupSubscription()
{
subscription = _streamStore.SubscribeToStream(
streamId: _streamId,
continueAfterVersion: version,
streamMessageReceived: StreamMessageReceived,
subscriptionDropped: SubscriptionDropped);
}
SetupSubscription();
return(subscription);
}
public async ValueTask <bool> MoveNextAsync()
{
ReadLoop:
if (await _inner.MoveNextAsync().ConfigureAwait(false))
{
return(true);
}
if (_cancellationToken.IsCancellationRequested)
{
return(false);
}
var currentPosition = _inner.CurrentPosition;
await _inner.DisposeAsync().ConfigureAwait(false);
Log.Verbose("Subscription {subscriptionId} to $all reached the end, switching...",
_subscriptionId);
if (_inner is LiveStreamSubscription)
{
_inner = new CatchupAllSubscription(_subscriptionId, _bus, currentPosition, _resolveLinks,
_user, _requiresLeader, _readIndex, _cancellationToken);
}
else
{
_inner = new LiveStreamSubscription(_subscriptionId, _bus, currentPosition, _resolveLinks,
_user, _requiresLeader, _cancellationToken);
}
goto ReadLoop;
}
public AllSubscriptionFiltered(IPublisher bus,
Position?startPosition,
bool resolveLinks,
IEventFilter eventFilter,
ClaimsPrincipal user,
bool requiresLeader,
IReadIndex readIndex,
uint?maxSearchWindow,
uint checkpointIntervalMultiplier,
Func <Position, Task> checkpointReached,
CancellationToken cancellationToken)
{
if (bus == null)
{
throw new ArgumentNullException(nameof(bus));
}
if (eventFilter == null)
{
throw new ArgumentNullException(nameof(eventFilter));
}
if (readIndex == null)
{
throw new ArgumentNullException(nameof(readIndex));
}
if (checkpointReached == null)
{
throw new ArgumentNullException(nameof(checkpointReached));
}
if (checkpointIntervalMultiplier == 0)
{
throw new ArgumentOutOfRangeException(nameof(checkpointIntervalMultiplier));
}
_subscriptionId = Guid.NewGuid();
_bus = bus;
_resolveLinks = resolveLinks;
_eventFilter = eventFilter;
_user = user;
_requiresLeader = requiresLeader;
_readIndex = readIndex;
_maxSearchWindow = maxSearchWindow;
_checkpointIntervalMultiplier = checkpointIntervalMultiplier;
_checkpointReached = checkpointReached;
_cancellationToken = cancellationToken;
_subscriptionStarted = new TaskCompletionSource <bool>();
_subscriptionStarted.SetResult(true);
_inner = startPosition == Position.End
? (IStreamSubscription) new LiveStreamSubscription(_subscriptionId, _bus,
Position.FromInt64(_readIndex.LastIndexedPosition, _readIndex.LastIndexedPosition),
_resolveLinks, _eventFilter, _user, _requiresLeader, _maxSearchWindow,
_checkpointIntervalMultiplier, checkpointReached, _cancellationToken)
: new CatchupAllSubscription(_subscriptionId, bus, startPosition ?? Position.Start, resolveLinks,
_eventFilter, user, _requiresLeader, readIndex, _maxSearchWindow, _checkpointIntervalMultiplier, checkpointReached,
cancellationToken);
}
public AllSubscription(IPublisher bus,
Position?startPosition,
bool resolveLinks,
ClaimsPrincipal user,
bool requiresLeader,
IReadIndex readIndex,
CancellationToken cancellationToken)
{
if (bus == null)
{
throw new ArgumentNullException(nameof(bus));
}
if (readIndex == null)
{
throw new ArgumentNullException(nameof(readIndex));
}
_subscriptionId = Guid.NewGuid();
_bus = bus;
_resolveLinks = resolveLinks;
_user = user;
_requiresLeader = requiresLeader;
_readIndex = readIndex;
_cancellationToken = cancellationToken;
_subscriptionStarted = new TaskCompletionSource <bool>();
_subscriptionStarted.SetResult(true);
_inner = startPosition == Position.End
? (IStreamSubscription) new LiveStreamSubscription(_subscriptionId, _bus,
Position.FromInt64(_readIndex.LastIndexedPosition, _readIndex.LastIndexedPosition),
_resolveLinks, _user, _requiresLeader, _cancellationToken)
: new CatchupAllSubscription(_subscriptionId, bus, startPosition ?? Position.Start, resolveLinks,
user, _requiresLeader, readIndex, cancellationToken);
}
private void SubscriptionDropped(IStreamSubscription subscription, SubscriptionDroppedReason reason,
Exception ex)
{
if (reason != SubscriptionDroppedReason.Disposed)
{
_log.FatalException("Subscription dropped", ex);
Environment.Exit(1);
}
}
private async Task OnReadEvent(IStreamSubscription arg1, ResolvedEvent arg2, CancellationToken t)
{
var eventData = Encoding.UTF8.GetString(arg2.Event.Data);
var metaData = Encoding.UTF8.GetString(arg2.Event.Metadata);
var ev = JsonConvert.DeserializeObject <TEvent>(eventData);
var m = JsonConvert.DeserializeObject <EventMetadata>(metaData);
_logger.Information("EventDispatcher is receiving {eventName}.", typeof(TEvent).Name);
await _dispatcher.Dispatch(m, ev);
}
public async Task Subscribe(IEventStoreFacade connection)
{
_connection = connection;
_lastCheckpoint = await _checkpointRepo.GetLastCheckpoint();
this.Subscription = await _connection.SubscribeToStreamAsync(_streamName,
new StreamRevision(_lastCheckpoint ?? 0L),
OnEventAppeared,
OnLiveProcessingStarted,
true,
OnSubscriptionDropped);
}
public async Task SubscribeToSocketAsync <T>(ExchangeChannel channel, IStreamSubscription subscription, Instrument?instrument = null) where T : ExchangeDto
{
if (!ExchangeConfig.SupportedWebsocketChannels.ContainsKey(channel))
{
return;
}
if (!_subscriptions.ContainsKey(channel))
{
_subscriptions.Add(channel, new List <IStreamSubscription>());
}
_subscriptions[channel].Add(subscription);
await _socketClient.SubscribeAsync(await GetSocketRequest(channel, instrument));
}
public ProjectionJob(
IEventStoreClient client,
IDependencyInitializer initializer,
IProjectionOptionsFactory optionsFactory,
IProjectionProcessor <TProjection> processor)
{
this.jobDependencies = initializer;
this.processor = processor;
var options = optionsFactory.GetOptions <TProjection>();
subscription = client.SubscribeToStreams(
ConsumerGroup.GetAsAutoScalingInstance(options.Name),
SubscriptionStartOptions.FromBegining,
OnProcessEventsAsync);
}
public CandleService(RepositoryService repoService, IEnumerable <IExchangeService> exchanges, ILogger logger)
{
_repoService = repoService;
_exchanges = exchanges.ToList();
_logger = logger;
_candleSubscription = new StreamSubscription <CandleModel>(InsertCandles);
_tradeSubscription = new StreamSubscription <TradeModel>(OnNewTrade);
_timePeriods = Enum.GetValues(typeof(TimePeriod)).Cast <TimePeriod>().ToArray();
_currentCandles = _timePeriods.ToDictionary(x => x,
y => _exchanges.ToDictionary(xx => xx.ExchangeConfig.ExchangeName,
yy => new CandleModel {
Period = y, ExchangeName = yy.ExchangeConfig.ExchangeName
}));
}
private async Task OnReadEvent(IStreamSubscription arg1, ResolvedEvent arg2, CancellationToken t)
{
var(m, ev) = _converter.Convert <TEvent>(arg2);
var groupName = typeof(TEvent).FullName.Replace(".", "-");
try
{
await _connection.Clients.All.SendCoreAsync(groupName, new object[] { m, ev });
Log.Information("SignalR hub send event {eventName} to it's clients.", typeof(TEvent).Name);
}
catch (Exception ex)
{
Debug.WriteLine(ex);
}
}
private async Task OnEventAppeared(IStreamSubscription arg1, ResolvedEvent arg2, CancellationToken t)
{
var eventData = Encoding.UTF8.GetString(arg2.Event.Data);
var metaData = Encoding.UTF8.GetString(arg2.Event.Metadata);
var ev = JsonConvert.DeserializeObject <TEvent>(eventData);
var m = JsonConvert.DeserializeObject <EventMetadata>(metaData);
m.Version = arg2.Event.EventNumber;
Log.Debug("Appeared {eventName} {version}@{streamName}", ev.GetType().Name, m.Version, arg2.Event.EventStreamId);
_lastCheckpoint = arg2.OriginalEventNumber;
await _coordinator.Push(_number, m, ev);
await _checkpointRepo.SaveCheckpoint(arg2.OriginalEventNumber);
}
private async Task StreamMessageReceived(IStreamSubscription subscription, StreamMessage streamMessage, CancellationToken cancellationToken)
{
switch (streamMessage.Type)
{
case "Deposited":
var depositedJson = await streamMessage.GetJsonData(cancellationToken);
var deposited = JsonConvert.DeserializeObject <Deposited>(depositedJson);
Balance = Balance.Add(deposited.Amount);
break;
case "Withdrawn":
var withdrawnJson = await streamMessage.GetJsonData(cancellationToken);
var withdrawn = JsonConvert.DeserializeObject <Withdrawn>(withdrawnJson);
Balance = Balance.Subtract(withdrawn.Amount);
break;
}
}
private async Task OnSpecEventAppearead <TEvent, TEventHandler>(IStreamSubscription arg1, ResolvedEvent arg2, CancellationToken arg3)
where TEvent : IEvent
where TEventHandler : IEventHandler <TEvent>
{
var eventData = Encoding.UTF8.GetString(arg2.Event.Data);
var metaData = Encoding.UTF8.GetString(arg2.Event.Metadata);
var ev = JsonConvert.DeserializeObject <TEvent>(eventData);
var m = JsonConvert.DeserializeObject <EventMetadata>(metaData);
//if (_continuum < m.TimeStamp)
// _continuum = m.TimeStamp;
//else throw new InvalidOperationException();
m.Version = arg2.Event.EventNumber;
using (var scope = _serviceProvider.CreateScope())
{
var handler = ActivatorUtilities.CreateInstance <TEventHandler>(scope.ServiceProvider);
await handler.Execute(m, ev);
}
}
public CommandProcessor(
IStreamStore streamStore,
ICommandProcessorPositionStore positionStore,
CommandHandlerDispatcher dispatcher,
Scheduler scheduler,
ILogger <CommandProcessor> logger)
{
if (streamStore == null)
{
throw new ArgumentNullException(nameof(streamStore));
}
if (positionStore == null)
{
throw new ArgumentNullException(nameof(positionStore));
}
if (dispatcher == null)
{
throw new ArgumentNullException(nameof(dispatcher));
}
_scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_messagePumpCancellation = new CancellationTokenSource();
_messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions
{
SingleReader = true,
SingleWriter = false,
AllowSynchronousContinuations = true
});
_messagePump = Task.Factory.StartNew(async() =>
{
IStreamSubscription subscription = null;
try
{
while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false))
{
while (_messageChannel.Reader.TryRead(out var message))
{
switch (message)
{
case Subscribe _:
logger.LogInformation("Subscribing ...");
subscription?.Dispose();
var version = await positionStore
.ReadVersion(RoadNetworkCommandQueue, _messagePumpCancellation.Token)
.ConfigureAwait(false);
logger.LogInformation("Subscribing as of {0}", version ?? -1);
subscription = streamStore.SubscribeToStream(
RoadNetworkCommandQueue,
version,
async(_, streamMessage, token) =>
{
var command = new ProcessStreamMessage(streamMessage);
await _messageChannel.Writer
.WriteAsync(command, token)
.ConfigureAwait(false);
await command
.Completion
.ConfigureAwait(false);
},
async(_, reason, exception) =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(
new SubscriptionDropped(reason, exception),
_messagePumpCancellation.Token)
.ConfigureAwait(false);
}
},
name: "RoadRegistry.BackOffice.CommandHost.CommandProcessor");
break;
case ProcessStreamMessage process:
try
{
logger.LogDebug(
"Processing {MessageType} at {Position}",
process.Message.Type, process.Message.Position);
var body = JsonConvert.DeserializeObject(
await process.Message
.GetJsonData(_messagePumpCancellation.Token)
.ConfigureAwait(false),
CommandMapping.GetEventType(process.Message.Type),
SerializerSettings);
var command = new Command(body).WithMessageId(process.Message.MessageId);
await dispatcher(command, _messagePumpCancellation.Token).ConfigureAwait(false);
await positionStore
.WriteVersion(RoadNetworkCommandQueue,
process.Message.StreamVersion,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
process.Complete();
}
catch (Exception exception)
{
_logger.LogError(exception, exception.Message);
// how are we going to recover from this? do we even need to recover from this?
// prediction: it's going to be a serialization error, a data quality error, or a bug
// if (process.Message.StreamVersion == 0)
// {
// await positionStore.WriteVersion(RoadNetworkCommandQueue,
// process.Message.StreamVersion,
// _messagePumpCancellation.Token);
// }
process.Fault(exception);
}
break;
case SubscriptionDropped dropped:
if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a stream store error.");
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Subscribe(), token).ConfigureAwait(false);
}
}, ResubscribeAfter);
}
else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a subscriber error.");
if (dropped.Exception != null &&
dropped.Exception is SqlException sqlException &&
sqlException.Number == -2 /* timeout */)
{
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Subscribe(), token).ConfigureAwait(false);
}
}, ResubscribeAfter);
}
}
break;
}
}
}
}
catch (TaskCanceledException)
{
if (logger.IsEnabled(LogLevel.Information))
{
logger.Log(LogLevel.Information, "CommandProcessor message pump is exiting due to cancellation.");
}
}
catch (OperationCanceledException)
{
if (logger.IsEnabled(LogLevel.Information))
{
logger.Log(LogLevel.Information, "CommandProcessor message pump is exiting due to cancellation.");
}
}
catch (Exception exception)
{
logger.LogError(exception, "CommandProcessor message pump is exiting due to a bug.");
}
finally
{
subscription?.Dispose();
}
}, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default);
}
public void Subscribe(IStreamStore streamStore, StreamId streamId)
{
StreamSubscription = streamStore.SubscribeToStream(streamId, null, StreamMessageReceived);
}
public SqlStreamStoreSubscription(IStreamSubscription sub, Func <bool> readyDelegate)
{
_readyDelegate = readyDelegate;
_subscription = sub;
}
private void OnDropped(IStreamSubscription arg1, SubscriptionDroppedReason arg2, Exception arg3)
{
}
private async Task StreamMessageReceived(IStreamSubscription subscription, StreamMessage streamMessage, CancellationToken cancellationToken)
{
var @event = await DeserializeJsonEvent(streamMessage, cancellationToken);
await _map.Handle(@event, Balance);
}
public Task SubscribeToStreamAsync <T>(ExchangeChannel channel, IStreamSubscription subscription, Instrument?instrument = null) where T : ExchangeDto
{
return(_exchangeApi.SubscribeToSocketAsync <T>(channel, subscription, instrument));
}
private void OnLiveProcessingStarted(IStreamSubscription obj)
{
_coordinator.ReceiverIsLive(_number);
}
private void OnSubscriptionDropped(IStreamSubscription arg1, SubscriptionDroppedReason arg2, Exception arg3)
{
//throw new NotImplementedException("We need to implement reconnect.");
_reconnectionCounter += 1;
}
