private async Task DoPoll()
{
if (Interlocked.CompareExchange(ref _isPolling, 1, 0) == 0)
{
try
{
var commits = _bucketId == null?
_persistStreams.GetFrom(_checkpointToken) :
_persistStreams.GetFrom(_bucketId, _checkpointToken);
await commits.ForEach(context =>
{
var commit = context.Item;
if (_stopRequested.IsCancellationRequested)
{
_subject.OnCompleted();
context.Break();
}
else
{
_subject.OnNext(commit);
_checkpointToken = commit.CheckpointToken;
}
return(Task.FromResult(false));
});
}
catch (Exception ex)
{
// These exceptions are expected to be transient
Logger.Error(ex.ToString());
}
Interlocked.Exchange(ref _isPolling, 0);
}
}
private void DoPoll()
{
if (Interlocked.CompareExchange(ref _isPolling, 1, 0) == 0)
{
try
{
var commits = _bucketId == null?
_persistStreams.GetFrom(_checkpointToken) :
_persistStreams.GetFrom(_bucketId, _checkpointToken);
foreach (var commit in commits)
{
if (_stopRequested.IsCancellationRequested)
{
_subject.OnCompleted();
return;
}
_subject.OnNext(commit);
_checkpointToken = commit.CheckpointToken;
}
}
catch (Exception ex)
{
// These exceptions are expected to be transient
Logger.Error(ex.ToString());
}
Interlocked.Exchange(ref _isPolling, 0);
}
}
public void Then_it_should_wait_for_the_polling_interval_to_retry()
{
A.CallTo(() => eventStore.GetFrom(A <string> .Ignored)).MustHaveHappened(Repeated.Exactly.Once);
utcNow = utcNow.Add(1.Seconds());
A.CallTo(() => eventStore.GetFrom(A <string> .Ignored)).MustHaveHappened(Repeated.Exactly.Once);
}
public void ConfigurePollingFunction(string bucketId = null)
{
if (bucketId == null)
{
_pollingFunc = () => _persistStreams.GetFrom(_checkpointToken);
}
else
{
_pollingFunc = () => _persistStreams.GetFrom(bucketId, _checkpointToken);
}
}
public void should_not_affect_the_stream_from_the_other_partition()
{
Commit[] stream = _persistence1.GetFrom(StreamId, 0, int.MaxValue).ToArray();
stream.ShouldNotBeNull();
stream.Count().ShouldBe(1);
stream.First().CommitStamp.ShouldBe(_attempt1.CommitStamp);
}
/// <summary>
/// Gets all commits after from start checkpoint.
/// </summary>
/// <param name="persistStreams">The IPersistStreams instance.</param>
public static IEnumerable <ICommit> GetFromStart(this IPersistStreams persistStreams)
{
if (persistStreams == null)
{
throw new ArgumentNullException(nameof(persistStreams));
}
return(persistStreams.GetFrom(0));
}
public static IEnumerable <ICommit> GetFrom(this IPersistStreams persistStreams, DateTime start)
{
if (persistStreams == null)
{
throw new ArgumentNullException(nameof(persistStreams));
}
return(persistStreams.GetFrom(Bucket.Default, start));
}
/// <summary>
/// Gets all commits after from start checkpoint.
/// </summary>
/// <param name="persistStreams">The IPersistStreams instance.</param>
public static IEnumerable <ICommit> GetFromStart(this IPersistStreams persistStreams)
{
if (persistStreams == null)
{
throw new ArgumentException("persistStreams is null");
}
return(persistStreams.GetFrom(null));
}
private IEnumerable <ICommit> GetCommitsFromPersistentStore(SendCommitAfterCurrentHeadCheckpointMessage msg)
{
IEnumerable <ICommit> commits = _persistStreams.GetFrom(msg.CurrentHeadCheckpoint.GetOrElse(() => 0)); //load all commits after checkpoint from db
if (!msg.NumberOfCommitsToSend.HasValue)
{
return(commits);
}
return(commits.Take(msg.NumberOfCommitsToSend.Get()));
}
public When_a_subscription_starts_after_zero_checkpoint_and_another_subscription_starts_after_null_checkpoint_while_the_first_subscription_is_loading_the_first_page_from_the_event_store()
{
Given(() =>
{
eventStore = A.Fake <IPersistStreams>();
A.CallTo(() => eventStore.GetFrom(A <string> .Ignored)).ReturnsLazily(call =>
{
string checkpointString = call.GetArgument <string>(0);
long checkpoint = string.IsNullOrEmpty(checkpointString)
? 0 :
long.Parse(checkpointString, CultureInfo.InvariantCulture);
aSubscriptionStartedLoading.Set();
if (!secondSubscriptionCreated.Wait(TimeSpan.FromSeconds(10)))
{
throw new InvalidOperationException("The second subscription has not been created in 10 seconds.");
}
// Give the second subscription enough time to access the cache.
Thread.Sleep(TimeSpan.FromSeconds(1));
return(checkpoint > 0
? new ICommit[0]
: new ICommit[] { new CommitBuilder().WithCheckpoint("1").Build() });
});
WithSubject(_ => new NEventStoreAdapter(eventStore, 100, pollingInterval, 100, () => utcNow));
});
When(() =>
{
Subject.Subscribe(0, transactions => Task.FromResult(0));
if (!aSubscriptionStartedLoading.Wait(TimeSpan.FromSeconds(10)))
{
throw new InvalidOperationException("The first subscription has not started loading in 10 seconds.");
}
Subject.Subscribe(
null,
transactions =>
{
secondSubscriptionReceivedTheTransaction.Set();
return(Task.FromResult(0));
});
secondSubscriptionCreated.Set();
});
}
public When_disposing_subscription()
{
Given(() =>
{
eventStore = A.Fake <IPersistStreams>();
A.CallTo(() => eventStore.GetFrom(A <string> .Ignored)).Returns(new ICommit[0]);
WithSubject(_ => new NEventStoreAdapter(eventStore, 11, pollingInterval, 100, () => utcNow));
subscription = Subject.Subscribe(1000, transactions => Task.FromResult(0));
});
When(() => subscription.Dispose(), deferedExecution: true);
}
private void GetNextCommits(CancellationToken cancellationToken)
{
IEnumerable <ICommit> commits = _persistStreams.GetFrom(_checkpointToken);
foreach (var commit in commits)
{
if (cancellationToken.IsCancellationRequested)
{
_subject.OnCompleted();
return;
}
_subject.OnNext(commit);
_checkpointToken = commit.CheckpointToken;
}
}
public void Unwind()
{
var lastEvent = _unwindedEventCollection
.Find(Builders <UnwindedDomainEvent> .Filter.Empty)
.Sort(Builders <UnwindedDomainEvent> .Sort.Descending(e => e.CheckpointToken))
.Limit(1)
.SingleOrDefault();
Int64 startToken = 0;
if (lastEvent != null)
{
startToken = lastEvent.CheckpointToken;
}
_logger.InfoFormat("Unwind events starting from commit {0}", startToken);
//Since we could have crashed during unwind of last commit, we want to reunwind again last commit
Int64 checkpointToken = startToken - 1;
_unwindedEventCollection.DeleteMany(Builders <UnwindedDomainEvent> .Filter.Gte(e => e.CheckpointToken, startToken));
Int64 count = 0;
List <UnwindedDomainEvent> batchEventUnwind = new List <UnwindedDomainEvent>(550);
foreach (var commit in _persistStream.GetFrom(checkpointToken))
{
foreach (var evt in UnwindCommit(commit))
{
batchEventUnwind.Add(evt);
if (batchEventUnwind.Count > 500)
{
_unwindedEventCollection.InsertMany(batchEventUnwind);
batchEventUnwind.Clear();
}
}
checkpointToken = commit.CheckpointToken;
if (++count % 10000 == 0)
{
_logger.InfoFormat("Unwinded block of 10000 commit, now we are at checkpoint {0}", checkpointToken);
}
}
if (batchEventUnwind.Count > 0)
{
_unwindedEventCollection.InsertMany(batchEventUnwind);
}
_logger.InfoFormat("Unwind events ends, started from commit {0} and ended with commit {1}", startToken, checkpointToken);
}
/// <summary>
/// Catches-up the projector if it has fallen behind the head
/// </summary>
protected virtual void Catchup()
{
var comparer = new CheckpointComparer();
IEnumerable <ICommit> commits = _persistStreams.GetFrom(Checkpoint.GetOrElse(() => 0)); //load all commits after our current checkpoint from db
foreach (var commit in commits)
{
Project(commit);
if (comparer.Compare(Checkpoint, commit.CheckpointToken.ToSome()) != 0)
{
//something went wrong, we couldn't project
Context.GetLogger().Warning("Stopped catchup! was unable to project the commit at checkpoint {0}", commit.CheckpointToken);
break;
}
}
}
public virtual IEnumerable <ICommit> GetFrom(string bucketId, string streamId, int minRevision, int maxRevision)
{
foreach (var commit in _persistence.GetFrom(bucketId, streamId, minRevision, maxRevision))
{
ICommit filtered = commit;
foreach (var hook in _pipelineHooks.Where(x => (filtered = x.Select(filtered)) == null))
{
Logger.Info(Resources.PipelineHookSkippedCommit, hook.GetType(), commit.CommitId);
break;
}
if (filtered == null)
{
Logger.Info(Resources.PipelineHookFilteredCommit);
}
else
{
yield return(filtered);
}
}
}
public When_there_are_no_more_commits()
{
Given(() =>
{
eventStore = A.Fake <IPersistStreams>();
A.CallTo(() => eventStore.GetFrom(A <string> .Ignored))
.Invokes(_ =>
{
eventStoreQueriedSource.SetResult(null);
})
.Returns(new ICommit[0]);
WithSubject(_ => new NEventStoreAdapter(eventStore, 11, pollingInterval, 100, () => utcNow));
Subject.Subscribe(1000, transactions => Task.FromResult(0));
});
When(async() =>
{
await eventStoreQueriedSource.Task;
});
}
public IEnumerable <ICommit> GetFrom(string bucketId, string streamId, int minRevision, int maxRevision)
{
return(underlying.GetFrom(bucketId, streamId, minRevision, maxRevision));
}
public IEnumerable <ICommit> GetFrom(string bucketId, string streamId, int minRevision, int maxRevision)
{
return(ExecuteHooks(_original.GetFrom(bucketId, streamId, minRevision, maxRevision)));
}
public virtual IEnumerable <Commit> GetFrom(string bucketId, string streamId, int minRevision, int maxRevision)
{
return(_persistence.GetFrom(bucketId, streamId, minRevision, maxRevision));
}
public IEnumerable <Commit> GetFrom(Guid streamId, int minRevision, int maxRevision)
{
return(original.GetFrom(streamId, minRevision, maxRevision));
}
private void DoPoll()
{
if (_persistStreams.IsDisposed)
{
return; //no need to poll, someone disposed NEventStore wihout sh
}
if (Interlocked.CompareExchange(ref _isPolling, 1, 0) == 0)
{
Int32 exceptionCount = 0;
do
{
try
{
skippedPoolCount = 0;
IEnumerable <ICommit> commits = _persistStreams.GetFrom(_checkpointToken);
Int64 i = 0;
foreach (var commit in commits)
{
if (_stopRequested.IsCancellationRequested)
{
_subject.OnCompleted();
return;
}
_enhancer.Enhance(commit);
_subject.OnNext(commit);
i++;
if (i % 1000 == 0)
{
var minCheckpoint = _tracker.GetMinCheckpoint();
var actualCheckpoint = Int64.Parse(commit.CheckpointToken);
while (actualCheckpoint > minCheckpoint + 1000)
{
Thread.Sleep(500);
minCheckpoint = _tracker.GetMinCheckpoint();
}
}
_checkpointToken = commit.CheckpointToken;
}
}
catch (Exception ex)
{
// These exceptions are expected to be transient
Logger.Error(ex.ToString());
exceptionCount++;
if (_persistStreams.IsDisposed || exceptionCount > 5)
{
//eventstore is stopped or too many exception, need to exit poll cycle.
skippedPoolCount = 0;
}
else
{
//conceptually if we have an exception and we are in manual poll we
//are potentially skipping some commit so we need to reschedule a poll
Interlocked.Increment(ref skippedPoolCount);
}
}
} while (skippedPoolCount > 0);
Interlocked.Exchange(ref _isPolling, 0);
}
else
{
Interlocked.Increment(ref skippedPoolCount);
Logger.Debug("Poll skipped, count " + skippedPoolCount);
}
}
private async Task <Page> TryLoadNextPage(long previousCheckpoint, string subscriptionId)
{
// Maybe it's just loaded to cache.
try
{
Page cachedPage = TryGetNextPageFromCache(previousCheckpoint, subscriptionId);
if (cachedPage.Transactions.Count > 0)
{
#if DEBUG
LogProvider.GetCurrentClassLogger()
.Debug(() =>
$"Loader for subscription {subscriptionId ?? "without ID"} has found a page in the cache.");
#endif
return(cachedPage);
}
}
catch (Exception exception)
{
LogProvider.GetLogger(typeof(NEventStoreAdapter))
.ErrorException(
$"Failed getting transactions after checkpoint {previousCheckpoint} from the cache.",
exception);
}
DateTime timeOfRequestUtc = getUtcNow();
List <Transaction> transactions;
try
{
transactions = await Task
.Run(() => eventStore
.GetFrom(previousCheckpoint.ToString())
.Take(maxPageSize)
.Select(ToTransaction)
.ToList())
.ConfigureAwait(false);
}
catch (Exception exception)
{
LogProvider.GetLogger(typeof(NEventStoreAdapter))
.ErrorException(
$"Failed loading transactions after checkpoint {previousCheckpoint} from NEventStore",
exception);
return(new Page(previousCheckpoint, new List <Transaction>()));
}
if (transactions.Count > 0)
{
#if DEBUG
LogProvider.GetCurrentClassLogger().Debug(() =>
$"Loader for subscription {subscriptionId ?? "without ID"} has loaded {transactions.Count} transactions " +
$"from checkpoint {transactions.First().Checkpoint} to checkpoint {transactions.Last().Checkpoint}.");
#endif
/* Add to cache in reverse order to prevent other projectors
* from requesting already loaded transactions which are not added to cache yet. */
for (int index = transactions.Count - 1; index > 0; index--)
{
transactionCacheByPreviousCheckpoint.Set(transactions[index - 1].Checkpoint, transactions[index]);
}
transactionCacheByPreviousCheckpoint.Set(previousCheckpoint, transactions[0]);
#if DEBUG
LogProvider.GetCurrentClassLogger().Debug(() =>
$"Loader for subscription {subscriptionId ?? "without ID"} has cached {transactions.Count} transactions " +
$"from checkpoint {transactions.First().Checkpoint} to checkpoint {transactions.Last().Checkpoint}.");
#endif
}
else
{
#if DEBUG
LogProvider.GetCurrentClassLogger().Debug(() =>
$"Loader for subscription {subscriptionId ?? "without ID"} has discovered " +
$"that there are no new transactions yet. Next request for the new transactions will be delayed.");
#endif
Volatile.Write(
ref lastExistingCheckpointRequest,
new CheckpointRequestTimestamp(previousCheckpoint, timeOfRequestUtc));
}
return(new Page(previousCheckpoint, transactions));
}
protected override void Because()
{
committed1 = persistence1.GetFrom(now).ToArray();
}
public void should_purge_all_commits_stored()
{
persistence1.GetFrom(DateTime.MinValue).Count().ShouldBe(0);
}
public IEnumerable <Transaction> GetFrom(long?checkpoint)
{
return(streamPersister
.GetFrom((!checkpoint.HasValue || checkpoint == 0) ? null : checkpoint.ToString())
.Select(ToTransaction));
}
private void DoPoll()
{
if (_persistStreams.IsDisposed)
{
return; //no need to poll, someone disposed NEventStore wihout sh
}
if (Interlocked.CompareExchange(ref _isPolling, 1, 0) == 0)
{
Int32 exceptionCount = 0;
do
{
try
{
skippedPoolCount = 0;
IEnumerable <ICommit> commits = _persistStreams.GetFrom(_checkpointToken);
foreach (var commit in commits)
{
if (_stopRequested.IsCancellationRequested)
{
_buffer.Complete();
return;
}
_enhancer.Enhance(commit);
while (!_buffer.SendAsync(commit).Wait(2000))
{
//maybe the mesh is full, but check for completion
if (_stopRequested.IsCancellationRequested)
{
_buffer.Complete();
return;
}
}
;
_checkpointToken = commit.CheckpointToken;
}
}
catch (Exception ex)
{
// These exceptions are expected to be transient
Logger.Error(ex.ToString());
exceptionCount++;
if (_persistStreams.IsDisposed || exceptionCount > 5)
{
//eventstore is stopped or too many exception, need to exit poll cycle.
skippedPoolCount = 0;
}
else
{
//conceptually if we have an exception and we are in manual poll we
//are potentially skipping some commit so we need to reschedule a poll
Interlocked.Increment(ref skippedPoolCount);
}
}
} while (skippedPoolCount > 0);
Interlocked.Exchange(ref _isPolling, 0);
}
else
{
Interlocked.Increment(ref skippedPoolCount);
Logger.Debug("Poll skipped, count " + skippedPoolCount);
}
}