/// <inheritdoc/>
public bool TryGetConsentFor(SubscriptionId subscriptionId, out ConsentId consentId)
{
var result = _subscriptions.TryGetValue(subscriptionId, out var subscription);
consentId = subscription?.ConsentId;
return(result);
}
/// <inheritdoc />
public bool TrySubscribe(
ConsentId consentId,
SubscriptionId subscriptionId,
EventProcessor eventProcessor,
EventsFromEventHorizonFetcher eventsFetcher,
CancellationToken cancellationToken,
out Subscription subscription)
{
subscription = default;
if (_subscriptions.ContainsKey(subscriptionId))
{
_logger.Warning("Subscription: '{SubscriptionId}' already registered", subscriptionId);
return(false);
}
subscription = new Subscription(
consentId,
subscriptionId,
eventProcessor,
eventsFetcher,
_streamProcessorStates,
() => Unregister(subscriptionId),
_eventsFetcherPolicy,
_loggerManager,
cancellationToken);
if (!_subscriptions.TryAdd(subscriptionId, subscription))
{
_logger.Warning("Subscription: '{SubscriptionId}' already registered", subscriptionId);
subscription = default;
return(false);
}
_logger.Trace("Subscription: '{SubscriptionId}' registered", subscriptionId);
return(true);
}
/// <inheritdoc/>
public async Task Write(CommittedEvent @event, ConsentId consentId, ScopeId scope, CancellationToken cancellationToken)
{
_logger.Trace(
"Writing Event Horizon Event: {EventLogSequenceNumber} from Tenant: {Tenant} in Microservice {Microservice} to Scope: {Scope}",
@event.EventLogSequenceNumber,
@event.ExecutionContext.Tenant,
@event.ExecutionContext.Microservice,
scope);
await _eventsToStreamsWriter.Write(
await _streams.GetEventLog(scope, cancellationToken).ConfigureAwait(false),
_eventFilter,
streamPosition => _eventConverter.ToEventLogEvent(
new CommittedExternalEvent(
streamPosition.Value,
@event.Occurred,
@event.EventSource,
@event.ExecutionContext,
@event.Type,
false,
@event.Content,
@event.EventLogSequenceNumber,
DateTimeOffset.UtcNow,
consentId)),
cancellationToken).ConfigureAwait(false);
}
void StartProcessingEventHorizon(
ConsentId consentId,
SubscriptionId subscriptionId,
MicroserviceAddress microserviceAddress,
IReverseCallClient <EventHorizonConsumerToProducerMessage, EventHorizonProducerToConsumerMessage, ConsumerSubscriptionRequest, Contracts.SubscriptionResponse, ConsumerRequest, ConsumerResponse> reverseCallClient)
{
Task.Run(async() =>
{
try
{
await ReadEventsFromEventHorizon(consentId, subscriptionId, reverseCallClient).ConfigureAwait(false);
throw new Todo(); // TODO: This is a hack to get the policy going. Remove this when we can have policies on return values
}
catch (Exception ex)
{
_logger.Debug(ex, "Reconnecting to event horizon with subscription {subscriptionId}", subscriptionId);
await _policy.Execute(
async _ =>
{
reverseCallClient = CreateClient(microserviceAddress, _cancellationToken);
var receivedResponse = await Subscribe(reverseCallClient, subscriptionId, microserviceAddress, _cancellationToken).ConfigureAwait(false);
var response = HandleSubscriptionResponse(receivedResponse, reverseCallClient.ConnectResponse, subscriptionId);
if (!response.Success)
{
throw new Todo(); // TODO: This is a hack to get the policy going. Remove this when we can have policies on return values
}
await ReadEventsFromEventHorizon(response.ConsentId, subscriptionId, reverseCallClient).ConfigureAwait(false);
throw new Todo(); // TODO: This is a hack to get the policy going. Remove this when we can have policies on return values
}, _cancellationToken).ConfigureAwait(false);
}
});
}
/// <inheritdoc/>
public async Task <EventLogSequenceNumber> Write(CommittedEvent @event, ConsentId consentId, ScopeId scope, CancellationToken cancellationToken)
{
_logger.WritingEventHorizonEvent(
@event.EventLogSequenceNumber,
@event.ExecutionContext.Tenant,
@event.ExecutionContext.Microservice,
scope);
var writtenStreamPosition = await _eventsToStreamsWriter.Write(
await _streams.GetEventLog(scope, cancellationToken).ConfigureAwait(false),
_eventFilter,
streamPosition => _eventConverter.ToEventLogEvent(
new CommittedExternalEvent(
streamPosition.Value,
@event.Occurred,
@event.EventSource,
@event.ExecutionContext,
@event.Type,
false,
@event.Content,
@event.EventLogSequenceNumber,
DateTimeOffset.UtcNow,
consentId)),
cancellationToken).ConfigureAwait(false);
_streamWatcher.NotifyForEvent(scope, StreamId.EventLog, writtenStreamPosition);
return(writtenStreamPosition.Value);
}
async Task ReadEventsFromEventHorizon(
ConsentId consentId,
SubscriptionId subscriptionId,
IReverseCallClient <EventHorizonConsumerToProducerMessage, EventHorizonProducerToConsumerMessage, ConsumerSubscriptionRequest, Contracts.SubscriptionResponse, ConsumerRequest, ConsumerResponse> reverseCallClient)
{
_logger.Information("Successfully connected event horizon with {subscriptionId}. Waiting for events to process", subscriptionId);
var queue = new AsyncProducerConsumerQueue <StreamEvent>();
var eventsFetcher = new EventsFromEventHorizonFetcher(queue);
using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken);
var tasks = new List <Task>();
try
{
_subscriptions.TrySubscribe(
consentId,
subscriptionId,
new EventProcessor(consentId, subscriptionId, _eventHorizonEventsWriter, _eventProcessorPolicy, _logger),
eventsFetcher,
linkedTokenSource.Token,
out var outputtedStreamProcessor);
using var streamProcessor = outputtedStreamProcessor;
await streamProcessor.Initialize().ConfigureAwait(false);
tasks.Add(Task.Run(async() =>
{
await reverseCallClient.Handle(
(request, cancellationToken) => HandleConsumerRequest(subscriptionId, queue, request, cancellationToken),
linkedTokenSource.Token).ConfigureAwait(false);
linkedTokenSource.Cancel();
}));
tasks.Add(streamProcessor.Start());
}
catch (Exception ex)
{
linkedTokenSource.Cancel();
_logger.Warning(ex, "Error occurred while initializing Subscription: {subscriptionId}", subscriptionId);
return;
}
var finishedTask = await Task.WhenAny(tasks).ConfigureAwait(false);
if (!linkedTokenSource.IsCancellationRequested)
{
linkedTokenSource.Cancel();
}
if (TryGetException(tasks, out var exception))
{
linkedTokenSource.Cancel();
_logger.Warning(exception, "Error occurred while processing Subscription: {subscriptionId}", subscriptionId);
}
await Task.WhenAll(tasks).ConfigureAwait(false);
}
/// <summary>
/// Initializes a new instance of the <see cref="EventProcessor"/> class.
/// </summary>
/// <param name="consentId">THe <see cref="ConsentId" />.</param>
/// <param name="subscription">The <see cref="Subscription" />.</param>
/// <param name="externalEventsCommitter">The <see cref="ICommitExternalEvents"/>.</param>
/// <param name="metrics">The system for collecting metrics.</param>
/// <param name="logger">The <see cref="ILogger" />.</param>
public EventProcessor(
ConsentId consentId,
SubscriptionId subscription,
ICommitExternalEvents externalEventsCommitter,
IMetricsCollector metrics,
ILogger logger)
{
_consentId = consentId;
Scope = subscription.ScopeId;
Identifier = subscription.ProducerTenantId.Value;
_subscriptionId = subscription;
_externalEventsCommitter = externalEventsCommitter;
_metrics = metrics;
_logger = logger;
}
/// <summary>
/// Initializes a new instance of the <see cref="EventProcessor"/> class.
/// </summary>
/// <param name="consentId">THe <see cref="ConsentId" />.</param>
/// <param name="subscription">The <see cref="Subscription" />.</param>
/// <param name="receivedEventsWriter">The <see cref="IWriteEventHorizonEvents" />.</param>
/// <param name="policy">The <see cref="IAsyncPolicyFor{T}" /> <see cref="EventProcessor" />.</param>
/// <param name="logger">The <see cref="ILogger" />.</param>
public EventProcessor(
ConsentId consentId,
SubscriptionId subscription,
IWriteEventHorizonEvents receivedEventsWriter,
IAsyncPolicyFor <EventProcessor> policy,
ILogger logger)
{
_consentId = consentId;
Scope = subscription.ScopeId;
Identifier = subscription.ProducerTenantId.Value;
_subscriptionId = subscription;
_receivedEventsWriter = receivedEventsWriter;
_policy = policy;
_logger = logger;
}
/// <summary>
/// Initializes a new instance of the <see cref="CommittedExternalEvent"/> class.
/// </summary>
/// <param name="eventLogSequenceNumber">The <see cref="EventLogSequenceNumber" />.</param>
/// <param name="occurred">The <see cref="DateTimeOffset" /> of when the Event was committed.</param>
/// <param name="eventSource">The <see cref="EventSourceId" />.</param>
/// <param name="executionContext">THe <see cref="ExecutionContext" />.</param>
/// <param name="type">The <see cref="Artifact" />.</param>
/// <param name="isPublic">Whether the event is public.</param>
/// <param name="content">The Event content.</param>
/// <param name="externalEventLogSequenceNumber">The external <see cref="EventLogSequenceNumber" />.</param>
/// <param name="received">The <see cref="DateTimeOffset" /> that this external Event was received.</param>
/// <param name="consent">The <see cref="Consent" />.</param>
public CommittedExternalEvent(
EventLogSequenceNumber eventLogSequenceNumber,
DateTimeOffset occurred,
EventSourceId eventSource,
ExecutionContext executionContext,
Artifact type,
bool isPublic,
string content,
EventLogSequenceNumber externalEventLogSequenceNumber,
DateTimeOffset received,
ConsentId consent)
: base(eventLogSequenceNumber, occurred, eventSource, executionContext, type, isPublic, content)
{
ExternalEventLogSequenceNumber = externalEventLogSequenceNumber;
Received = received;
Consent = consent;
}
/// <inheritdoc/>
public IStreamProcessor Create(
ConsentId consent,
SubscriptionId subscription,
ExecutionContext executionContext,
EventsFromEventHorizonFetcher eventsFromEventHorizonFetcher)
=> new StreamProcessor(
subscription,
executionContext,
new EventProcessor(
consent,
subscription,
_externalEventsCommitter,
_metrics,
_loggerFactory.CreateLogger <EventProcessor>()),
eventsFromEventHorizonFetcher,
_streamProcessorStates,
_eventsFetcherPolicy,
_metrics,
_loggerFactory);
bool TryGetConsent(ConsumerSubscriptionArguments arguments, out ConsentId consent, out SubscriptionResponse failureResponse)
{
consent = default;
try
{
return(TryGetEventHorizonsConfiguration(arguments, out var eventHorizonsConfiguration, out failureResponse) &&
TryGetConsentFromConfiguration(arguments, eventHorizonsConfiguration, out consent, out failureResponse));
}
catch (Exception ex)
{
Log.ErrorCreatingSubscriptionResponse(_logger, ex);
failureResponse = new SubscriptionResponse
{
Failure = new ProtobufContracts.Failure
{
Id = FailureId.Other.ToProtobuf(),
Reason = "Error occurred while creating subscription response"
}
};
return(false);
}
}
/// <inheritdoc />
public async Task Commit(CommittedEvents events, ConsentId consent, ScopeId scope)
{
foreach (var @event in events)
{
var sequenceNumber = await _policies.WriteEvent.ExecuteAsync(
_ => _receivedEventsWriter.Write(@event, consent, scope, CancellationToken.None),
CancellationToken.None).ConfigureAwait(false);
await _eventStoreClient.CommitExternal(new CommitExternalEventsRequest
{
ScopeId = scope.ToProtobuf(),
Event = new CommittedEvent(
sequenceNumber,
@event.Occurred,
@event.EventSource,
@event.ExecutionContext,
@event.Type,
@event.Public,
@event.Content).ToProtobuf()
}, CancellationToken.None).ConfigureAwait(false);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="EventHorizon"/> class.
/// </summary>
/// <param name="id">The <see cref="EventHorizonId"/>.</param>
/// <param name="consent">The <see cref="ConsentId"/>.</param>
/// <param name="currentPosition">The initial current <see cref="StreamPosition"/> of the event horizon.</param>
/// <param name="dispatcher">The reverse call dispatcher.</param>
/// <param name="executionContexts">The <see cref="ICreateExecutionContexts"/>.</param>
/// <param name="getEventFetchers">The <see cref="Func{TResult}"/> callback for getting <see cref="IEventFetchers"/> in a tenant container.</param>
/// <param name="getStreamWaiter">The <see cref="Func{TResult}"/> callback for getting <see cref="IStreamEventWatcher"/> in a tenant container</param>
/// <param name="metrics">The <see cref="IMetricsCollector"/>.</param>
/// <param name="logger">The <see cref="ILogger"/>.</param>
/// <param name="cancellationToken">The cancellation token.</param>
public EventHorizon(
EventHorizonId id,
ConsentId consent,
StreamPosition currentPosition,
IReverseCallDispatcher <EventHorizonConsumerToProducerMessage, EventHorizonProducerToConsumerMessage, ConsumerSubscriptionRequest, SubscriptionResponse, ConsumerRequest, ConsumerResponse> dispatcher,
ICreateExecutionContexts executionContexts,
Func <TenantId, IEventFetchers> getEventFetchers,
Func <TenantId, IStreamEventWatcher> getStreamWaiter,
IMetricsCollector metrics,
ILogger logger,
CancellationToken cancellationToken)
{
Consent = consent;
_dispatcher = dispatcher;
_executionContexts = executionContexts;
_getEventFetchers = getEventFetchers;
_getStreamWaiter = getStreamWaiter;
_metrics = metrics;
_logger = logger;
Id = id;
CurrentPosition = currentPosition;
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
}
/// <summary>
/// Initializes a new instance of the <see cref="Subscription"/> class.
/// </summary>
/// <param name="consentId">The <see cref="ConsentId" />.</param>
/// <param name="subscriptionId">The <see cref="StreamProcessorId" />.</param>
/// <param name="eventProcessor">The <see cref="IEventProcessor" />.</param>
/// <param name="eventsFetcher">The <see cref="EventsFromEventHorizonFetcher" />.</param>
/// <param name="streamProcessorStates">The <see cref="IResilientStreamProcessorStateRepository" />.</param>
/// <param name="unregister">An <see cref="Action" /> that unregisters the <see cref="ScopedStreamProcessor" />.</param>
/// <param name="eventsFetcherPolicy">The <see cref="IAsyncPolicyFor{T}" /> <see cref="ICanFetchEventsFromStream" />.</param>
/// <param name="loggerManager">The <see cref="ILoggerManager" />.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken" />.</param>
public Subscription(
ConsentId consentId,
SubscriptionId subscriptionId,
EventProcessor eventProcessor,
EventsFromEventHorizonFetcher eventsFetcher,
IResilientStreamProcessorStateRepository streamProcessorStates,
Action unregister,
IAsyncPolicyFor <ICanFetchEventsFromStream> eventsFetcherPolicy,
ILoggerManager loggerManager,
CancellationToken cancellationToken)
{
_identifier = subscriptionId;
_eventProcessor = eventProcessor;
_unregister = unregister;
_streamProcessorStates = streamProcessorStates;
_eventsFetcher = eventsFetcher;
_eventsFetcherPolicy = eventsFetcherPolicy;
_loggerManager = loggerManager;
_logger = loggerManager.CreateLogger <StreamProcessor>();
_cancellationToken = cancellationToken;
_unregisterTokenRegistration = _cancellationToken.Register(_unregister);
ConsentId = consentId;
}
bool TryGetConsentFromConfiguration(
ConsumerSubscriptionArguments arguments,
EventHorizonsPerMicroserviceConfiguration eventHorizonsConfiguration,
out ConsentId consentId,
out SubscriptionResponse failureResponse)
{
consentId = default;
failureResponse = default;
Log.CheckingConsents(
_logger,
arguments.Partition,
arguments.PublicStream,
arguments.ProducerTenant,
arguments.ConsumerTenant,
arguments.ConsumerMicroservice);
if (!eventHorizonsConfiguration.TryGetValue(arguments.ConsumerMicroservice, out var eventHorizonConfiguration))
{
failureResponse = CreateNoConsentsConfiguredResponse(
arguments,
$"There are no consents configured for Consumer Microservice {arguments.ConsumerMicroservice}");
return(false);
}
foreach (var consent in eventHorizonConfiguration.Consents)
{
if (consent.ConsumerTenant == arguments.ConsumerTenant.Value && consent.Stream == arguments.PublicStream.Value && consent.Partition == arguments.Partition.Value)
{
consentId = consent.Consent;
return(true);
}
}
failureResponse = CreateNoConsentsConfiguredResponse(
arguments,
$"There are no consents configured for Consumer Tenant {arguments.ConsumerTenant} from Public Stream {arguments.PublicStream} and Partition {arguments.Partition}");
return(false);
}
internal static void SubscriptionIsReceivingAndWriting(this ILogger logger, SubscriptionId subscriptionId, ConsentId consentId) => _subscriptionIsReceivingAndWriting(logger, consentId, subscriptionId, null);
bool TryGetConsentFor(Microservice consumerMicroservice, TenantId consumerTenant, TenantId producerTenant, StreamId publicStream, PartitionId partition, out ConsentId consentId)
{
consentId = null;
_logger.Trace(
"Checking consents configured for Partition: {Partition} in Public Stream {PublicStream} in Tenant {ProducerTenant} to Consumer Tenant {ConsumerTenant} in Microservice {ConsumerMicroservice}",
partition,
publicStream,
producerTenant,
consumerTenant,
consumerMicroservice);
var consentsForSubscription = _eventHorizonConsents
.GetConsentConfigurationsFor(producerTenant)
.Where(_ => _.Microservice == consumerMicroservice && _.Tenant == consumerTenant && _.Stream == publicStream && _.Partition == partition).ToArray();
if (consentsForSubscription.Length == 0)
{
_logger.Debug(
"There are no consent configured for partition '{Partition}' in public stream '{PublicStream}' in tenant '{ProducerTenant}' to consumer tenant '{ConsumerTenant}' in microservice '{ConsumerMicroservice}'",
partition,
publicStream,
producerTenant,
consumerTenant,
consumerMicroservice);
return(false);
}
if (consentsForSubscription.Length > 1)
{
_logger.Warning(
"There are multiple consents configured for Partition {Partition} in Public Stream {PublicStream} in Tenant {ProducerTenant} to Consumer Tenant {ConsumerTenant} in Microservice {ConsumerMicroservice}",
partition,
publicStream,
producerTenant,
consumerTenant,
consumerMicroservice);
}
consentId = consentsForSubscription.SingleOrDefault()?.Consent;
return(consentId != null);
}
/// <summary>
/// Initializes a new instance of the <see cref="SuccessfulSubscriptionResponse"/> class.
/// </summary>
/// <param name="consentId">The <see cref="ConsentId" />.</param>
public SuccessfulSubscriptionResponse(ConsentId consentId)
: base(consentId)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionResponse"/> class.
/// </summary>
/// <param name="consentId">The <see cref="ConsentId" />.</param>
SubscriptionResponse(ConsentId consentId)
{
Success = true;
ConsentId = consentId;
}
internal static void SubsciptionFailedWhileReceivingAndWriting(this ILogger logger, SubscriptionId subscriptionId, ConsentId consentId, Exception exception) => _subsciptionFailedWhileReceivingAndWriting(logger, consentId, subscriptionId, exception);