public async Task CanDropAndConnectInDifferentMode(SubscriptionOpeningStrategy strategy1, SubscriptionOpeningStrategy strategy2)
{
using (var store = GetDocumentStore())
{
var id = store.Subscriptions.Create <User>();
using var subscription1 = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(id)
{
Strategy = strategy1,
});
using var subscription2 = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(id)
{
Strategy = strategy2,
});
var mre1 = new ManualResetEventSlim();
var mre2 = new ManualResetEventSlim();
subscription1.OnEstablishedSubscriptionConnection += mre1.Set;
subscription2.OnEstablishedSubscriptionConnection += mre2.Set;
var t = subscription1.Run(x =>
{
});
Assert.True(mre1.Wait(TimeSpan.FromSeconds(15)));
mre1.Reset();
await store.Subscriptions.DropSubscriptionWorkerAsync(subscription1);
await Assert.ThrowsAsync <SubscriptionClosedException>(() => t);
t = subscription2.Run((_) => { });
Assert.True(mre2.Wait(TimeSpan.FromSeconds(15)));
}
}
private static async Task CreateWorker(DocumentStore store, string subscriptionName, SubscriptionOpeningStrategy strategy = SubscriptionOpeningStrategy.WaitForFree, CancellationToken token = default)
{
while (token.IsCancellationRequested == false)
{
try
{
using (var subscription = store.Subscriptions.GetSubscriptionWorker <User>(new SubscriptionWorkerOptions(subscriptionName)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromMilliseconds(500),
MaxDocsPerBatch = 5,
MaxErroneousPeriod = TimeSpan.FromSeconds(5),
Strategy = strategy,
}))
{
await subscription.Run((async batch =>
{
if (Interlocked.Increment(ref _batchCount) % 11 == 0)
{
throw new InvalidOperationException("Random sub failure before");
}
using (var session = batch.OpenAsyncSession())
{
foreach (var item in batch.Items)
{
item.Result.Count--;
}
await session.SaveChangesAsync(token);
}
if (Interlocked.Increment(ref _batchCount) % 7 == 0)
{
throw new InvalidOperationException("Random sub failure after");
}
}));
}
}
catch
{
await Task.Delay(500, token);
}
}
}
public void ShouldWaitForClientToProcessCurrentBatchBeforeTakingOver(SubscriptionOpeningStrategy openingStrategy)
{
using (var store = NewDocumentStore())
{
// fill in database to make sure first subscription has something to process
using (var s = store.OpenSession())
{
s.Store(new User());
s.Store(new User());
s.SaveChanges();
}
var id = store.Subscriptions.Create(new SubscriptionCriteria <User>());
var subscription = store.Subscriptions.Open <User>(id, new SubscriptionConnectionOptions()
{
Strategy = SubscriptionOpeningStrategy.OpenIfFree
});
store.Changes().WaitForAllPendingSubscriptions();
var items = new BlockingCollection <User>();
var firstSubscriptionStartedProcessingMre = new ManualResetEvent(false);
subscription.Subscribe(item =>
{
firstSubscriptionStartedProcessingMre.Set();
Thread.Sleep(500);
items.Add(item);
});
// we want to make sure that we open second subscription when first one in processed
Assert.True(firstSubscriptionStartedProcessingMre.WaitOne(TimeSpan.FromSeconds(10)));
var forcedSubscription = store.Subscriptions.Open <User>(id, new SubscriptionConnectionOptions()
{
Strategy = openingStrategy
});
store.Changes().WaitForAllPendingSubscriptions();
var forcedItems = new BlockingCollection <User>();
forcedSubscription.Subscribe(forcedItems.Add);
User user;
// first subscription should contain 2 elements
Assert.True(items.TryTake(out user, waitForDocTimeout));
Assert.True(items.TryTake(out user, waitForDocTimeout));
Assert.False(items.TryTake(out user, TimeSpan.FromSeconds(1)));
// forcedItems should be empty at this point as server should wait for first subscription
// to complete, thus leaving no records to process for second subscription
Assert.False(forcedItems.TryTake(out user, waitForDocTimeout));
Assert.True(SpinWait.SpinUntil(() => subscription.IsConnectionClosed, TimeSpan.FromSeconds(5)));
Assert.True(subscription.SubscriptionConnectionException is SubscriptionInUseException);
// now fill database with extra records
using (var s = store.OpenSession())
{
s.Store(new User());
s.Store(new User());
s.SaveChanges();
}
// only second subscription should get results
Assert.True(forcedItems.TryTake(out user, waitForDocTimeout));
Assert.True(forcedItems.TryTake(out user, waitForDocTimeout));
Assert.False(forcedItems.TryTake(out user, TimeSpan.FromSeconds(1)));
Assert.False(items.TryTake(out user, TimeSpan.FromSeconds(1)));
}
}
public void CanTakeOverEvenAfterPreviousSubscriptionTimedOut(SubscriptionOpeningStrategy openingStrategy)
{
using (var store = NewDocumentStore())
{
// fill in database to make sure first subscription has something to process
using (var s = store.OpenSession())
{
s.Store(new User());
s.Store(new User());
s.SaveChanges();
}
var id = store.Subscriptions.Create(new SubscriptionCriteria <User>());
var subscription = store.Subscriptions.Open <User>(id, new SubscriptionConnectionOptions
{
Strategy = SubscriptionOpeningStrategy.OpenIfFree,
BatchOptions = new SubscriptionBatchOptions
{
AcknowledgmentTimeout = TimeSpan.FromSeconds(5)
}
});
store.Changes().WaitForAllPendingSubscriptions();
var items = new BlockingCollection <User>();
var firstSubscriptionStartedProcessingMre = new ManualResetEvent(false);
subscription.Subscribe(item =>
{
firstSubscriptionStartedProcessingMre.Set();
Thread.Sleep(3000); // 2 objects * 3000 gives to 6 second to process, so we can't ack batch
items.Add(item);
});
// we want to make sure that we open second subscription when first one in processed
Assert.True(firstSubscriptionStartedProcessingMre.WaitOne(TimeSpan.FromSeconds(10)));
var forcedSubscription = store.Subscriptions.Open <User>(id, new SubscriptionConnectionOptions()
{
Strategy = openingStrategy
});
store.Changes().WaitForAllPendingSubscriptions();
var forcedItems = new BlockingCollection <User>();
forcedSubscription.Subscribe(forcedItems.Add);
User user;
// first subscription should contain 2 elements - how ever this batch won't be acknowledged
Assert.True(items.TryTake(out user, waitForDocTimeout));
Assert.True(items.TryTake(out user, waitForDocTimeout));
Assert.False(items.TryTake(out user, TimeSpan.FromSeconds(1)));
// forcedItems should contain also 2 elements because batch in first subscription was not acknowledged on time
Assert.True(forcedItems.TryTake(out user, waitForDocTimeout));
Assert.True(forcedItems.TryTake(out user, waitForDocTimeout));
Assert.False(forcedItems.TryTake(out user, TimeSpan.FromSeconds(1)));
Assert.True(SpinWait.SpinUntil(() => subscription.IsConnectionClosed, TimeSpan.FromSeconds(5)));
Assert.True(subscription.SubscriptionConnectionException is SubscriptionInUseException);
}
}
public void RecordConnectionInfo(SubscriptionState subscriptionState, string clientUri, SubscriptionOpeningStrategy strategy, string workerId)
{
_stats.TaskId = subscriptionState.SubscriptionId;
_stats.TaskName = subscriptionState.SubscriptionName;
_stats.ClientUri = clientUri;
_stats.Strategy = strategy;
_stats.WorkerId = workerId;
}