public void CanNameAndOpenWithNameOnly()
{
using (var store = GetDocumentStore())
{
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Name = "get-users",
};
var subscriptionId = store.Subscriptions.Create <User>(options: subscriptionCreationParams);
var subscription = store.Subscriptions.Open <User>(new SubscriptionConnectionOptions("get-users"));
var users = new BlockingCollection <User>();
using (var session = store.OpenSession())
{
session.Store(new User());
session.SaveChanges();
}
var subscirptionLifetimeTask = subscription.Run(u =>
{
foreach (var item in u.Items)
{
users.Add(item.Result);
}
});
User user;
Assert.True(users.TryTake(out user, 1000000));
}
}
public void GetSubscriptionTaskInfo()
{
var subscriptionOption = new SubscriptionCreationOptions <Query.Order>
{
Name = "sub",
Filter = x => x.Employee.StartsWith("e"),
MentorNode = "B"
};
using (var store = GetDocumentStore())
{
var sub = store.Subscriptions.Create(subscriptionOption);
var op = new GetOngoingTaskInfoOperation(subscriptionOption.Name, OngoingTaskType.Subscription);
var subscriptionResult = (OngoingTaskSubscription)store.Maintenance.Send(op);
Assert.Equal(subscriptionOption.MentorNode, subscriptionResult.MentorNode);
Assert.NotNull(subscriptionResult.Query);
var state = store.Subscriptions.GetSubscriptionState(subscriptionOption.Name);
op = new GetOngoingTaskInfoOperation(state.SubscriptionId, OngoingTaskType.Subscription);
subscriptionResult = (OngoingTaskSubscription)store.Maintenance.Send(op);
Assert.Equal(subscriptionOption.MentorNode, subscriptionResult.MentorNode);
Assert.Equal(subscriptionOption.Name, subscriptionResult.SubscriptionName);
Assert.NotNull(subscriptionResult.Query);
op = new GetOngoingTaskInfoOperation(state.SubscriptionId - 1, OngoingTaskType.Subscription);
Assert.Throws <SubscriptionDoesNotExistException>(() => store.Maintenance.Send(op));
}
}
public async Task BasicSusbscriptionTest()
{
using (var store = GetDocumentStore())
{
await CreateDocuments(store, 1);
var lastChangeVector = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector;
await CreateDocuments(store, 5);
var subscriptionCreationParams = new SubscriptionCreationOptions()
{
Query = "from Things",
ChangeVector = lastChangeVector
};
var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
using (var subscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId)))
{
var list = new BlockingCollection <Thing>();
GC.KeepAlive(subscription.Run(u =>
{
foreach (var item in u.Items)
{
list.Add(item.Result);
}
}));
Thing thing;
for (var i = 0; i < 5; i++)
{
Assert.True(list.TryTake(out thing, 1000));
}
Assert.False(list.TryTake(out thing, 50));
}
}
}
public async Task <long> PutSubscription(SubscriptionCreationOptions options, string raftRequestId, long?subscriptionId = null, bool?disabled = false, string mentor = null)
{
var command = new PutSubscriptionCommand(_db.Name, options.Query, mentor, raftRequestId)
{
InitialChangeVector = options.ChangeVector,
SubscriptionName = options.Name,
SubscriptionId = subscriptionId,
Disabled = disabled ?? false
};
var(etag, _) = await _serverStore.SendToLeaderAsync(command);
if (_logger.IsInfoEnabled)
{
_logger.Info($"New Subscription with index {etag} was created");
}
await _db.RachisLogIndexNotifications.WaitForIndexNotification(etag, _serverStore.Engine.OperationTimeout);
if (subscriptionId != null)
{
// updated existing subscription
return(subscriptionId.Value);
}
return(etag);
}
public async Task BasicCriteriaTest(bool useSsl)
{
string dbName = GetDatabaseName();
X509Certificate2 clientCertificate = null;
X509Certificate2 adminCertificate = null;
if (useSsl)
{
var certificates = Certificates.SetupServerAuthentication();
adminCertificate = Certificates.RegisterClientCertificate(certificates.ServerCertificate.Value, certificates.ClientCertificate1.Value, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin);
clientCertificate = Certificates.RegisterClientCertificate(certificates.ServerCertificate.Value, certificates.ClientCertificate2.Value, new Dictionary <string, DatabaseAccess>
{
[dbName] = DatabaseAccess.ReadWrite
});
}
using (var store = GetDocumentStore(new Options
{
AdminCertificate = adminCertificate,
ClientCertificate = clientCertificate,
ModifyDatabaseName = s => dbName
}))
{
using (var subscriptionManager = new DocumentSubscriptions(store))
{
await CreateDocuments(store, 1);
var lastChangeVector = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector;
await CreateDocuments(store, 5);
var subscriptionCreationParams = new SubscriptionCreationOptions()
{
Query = "from Things where Name = 'ThingNo3'",
ChangeVector = lastChangeVector
};
var subsId = subscriptionManager.Create(subscriptionCreationParams);
using (var subscription = subscriptionManager.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var list = new BlockingCollection <Thing>();
GC.KeepAlive(subscription.Run(x =>
{
foreach (var item in x.Items)
{
list.Add(item.Result);
}
}));
Thing thing;
Assert.True(list.TryTake(out thing, _reasonableWaitTime));
Assert.Equal("ThingNo3", thing.Name);
Assert.False(list.TryTake(out thing, 50));
}
}
}
}
private static bool SubscriptionHasChanges(SubscriptionCreationOptions options, SubscriptionState state)
{
bool gotChanges = options.Name != state.SubscriptionName ||
options.ChangeVector != state.ChangeVectorForNextBatchStartingPoint ||
options.MentorNode != state.MentorNode ||
options.Query != state.Query;
return(gotChanges);
}
public async Task BasicSusbscriptionTest()
{
using (var store = GetDocumentStore())
{
await CreateDocuments(store, 1);
var lastChangeVector = store.Maintenance.Send(new GetStatisticsOperation()).DatabaseChangeVector;
await CreateDocuments(store, 5);
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from Things",
ChangeVector = lastChangeVector
};
var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false);
using (var subscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var bc = new BlockingCollection <Thing>();
GC.KeepAlive(subscription.Run(x =>
{
foreach (var item in x.Items)
{
bc.Add(item.Result);
}
}));
Thing thing;
for (var i = 0; i < 5; i++)
{
Assert.True(bc.TryTake(out thing, 1000));
}
Assert.False(bc.TryTake(out thing, 50));
for (var j = 0; j < 2; j++)
{
await CreateDocuments(store, 1);
Assert.True(bc.TryTake(out thing, 500));
Assert.False(bc.TryTake(out thing, 50));
}
}
}
}
private async Task <RunResult> SingleTestRun()
{
try
{
if (string.IsNullOrEmpty(_subscriptionName))
{
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from " + _collectionName
};
_subscriptionName = await _store.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false);
}
using (var subscription = _store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(_subscriptionName)
{
Strategy = SubscriptionOpeningStrategy.WaitForFree,
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var tcs = new TaskCompletionSource <object>();
var sp = Stopwatch.StartNew();
int count = 0;
var task = subscription.Run(o =>
{
if (count++ >= _batchSize)
{
tcs.TrySetResult(null);
}
});;
await tcs.Task.ConfigureAwait(false);
await subscription.DisposeAsync().ConfigureAwait(false);
await task;
return(new RunResult
{
DocsProccessed = count,
DocsRequested = _batchSize,
ElapsedMs = sp.ElapsedMilliseconds
});
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
throw;
}
}
public async Task ShouldRespectCollectionCriteria()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 100; i++)
{
await session.StoreAsync(new Company());
await session.StoreAsync(new User());
await session.StoreAsync(new Address());
}
await session.SaveChangesAsync();
}
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from Users"
};
var id = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
using (var subscription = store.Subscriptions.GetSubscriptionWorker(
new SubscriptionWorkerOptions(id)
{
MaxDocsPerBatch = 31,
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var ids = new List <string>();
GC.KeepAlive(subscription.Run(batch =>
{
foreach (var item in batch.Items)
{
ids.Add(item.Id);
}
}));
Assert.True(SpinWait.SpinUntil(() => ids.Count >= 100, TimeSpan.FromSeconds(60)));
Assert.Equal(100, ids.Count);
foreach (var i in ids)
{
Assert.True(i.StartsWith("users/"));
}
}
}
}
private async Task CreateInternal(BlittableJsonReaderObject bjro, SubscriptionCreationOptions options, DocumentsOperationContext context, long?id, bool?disabled)
{
if (TrafficWatchManager.HasRegisteredClients)
{
AddStringToHttpContext(bjro.ToString(), TrafficWatchChangeType.Subscriptions);
}
var sub = SubscriptionConnection.ParseSubscriptionQuery(options.Query);
if (Enum.TryParse(options.ChangeVector, out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
{
switch (changeVectorSpecialValue)
{
case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
options.ChangeVector = null;
break;
case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
options.ChangeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, sub.Collection);
break;
}
}
var mentor = options.MentorNode;
var subscriptionId = await Database.SubscriptionStorage.PutSubscription(options, GetRaftRequestIdFromQuery(), id, disabled, mentor);
var name = options.Name ?? subscriptionId.ToString();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext))
using (serverContext.OpenReadTransaction())
{
// need to wait on the relevant remote node
var node = Database.SubscriptionStorage.GetResponsibleNode(serverContext, name);
if (node != null && node != ServerStore.NodeTag)
{
await WaitForExecutionOnSpecificNode(serverContext, ServerStore.GetClusterTopology(serverContext), node, subscriptionId);
}
}
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(CreateSubscriptionResult.Name)] = name
});
}
}
public void ShouldStopPullingTaskWhenSubscriptionIsDeleted()
{
using (var store = GetDocumentStore())
{
// insert few documents and fetch them using subscription
using (var session = store.OpenSession())
{
for (int i = 0; i < 10; i++)
{
session.Store(new Company());
}
session.SaveChanges();
}
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from Companies"
};
var id = store.Subscriptions.Create(subscriptionCreationParams);
var subscription = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(id)
{
MaxDocsPerBatch = 5,
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
});
var docs = new List <dynamic>();
GC.KeepAlive(subscription.Run(batch => docs.AddRange(batch.Items.Select(item => item.Result))));
Assert.True(SpinWait.SpinUntil(() => docs.Count == 10, TimeSpan.FromSeconds(60)));
// all documents were fetched - time to delete subscription
store.Subscriptions.DeleteAsync(id).Wait();
// verify if we don't get new items
using (var session = store.OpenSession())
{
for (int i = 0; i < 2; i++)
{
session.Store(new Company());
}
session.SaveChanges();
}
// wait 3 seconds for new documents - we shouldn't get any
Assert.False(SpinWait.SpinUntil(() => docs.Count != 10, TimeSpan.FromSeconds(3)));
}
}
public async Task <long> PutSubscription(SubscriptionCreationOptions options, long?subscriptionId = null, bool?disabled = false)
{
var command = new PutSubscriptionCommand(_db.Name, options.Query)
{
InitialChangeVector = options.ChangeVector,
SubscriptionName = options.Name,
SubscriptionId = subscriptionId,
Disabled = disabled ?? false
};
var(etag, _) = await _serverStore.SendToLeaderAsync(command);
if (_logger.IsInfoEnabled)
{
_logger.Info($"New Subscription with index {etag} was created");
}
await _db.RachisLogIndexNotifications.WaitForIndexNotification(etag);
return(etag);
}
public async Task CreateSubscription()
{
using (var store = GetDocumentStore())
{
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from People"
};
var subsName = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
var subscriptionsConfig = await store.Subscriptions.GetSubscriptionsAsync(0, 10);
var subscripitonState = await store.Subscriptions.GetSubscriptionStateAsync(subsName);
Assert.Equal(1, subscriptionsConfig.Count);
Assert.Equal(subscriptionCreationParams.Query, subscriptionsConfig[0].Query);
Assert.Null(subscriptionsConfig[0].ChangeVectorForNextBatchStartingPoint);
Assert.Equal(subsName, subscriptionsConfig[0].SubscriptionName);
Assert.Equal(subscripitonState.SubscriptionId, subscriptionsConfig[0].SubscriptionId);
}
}
public async Task Subscription_WhenFilteredByNull_ShouldWork()
{
const string id = "A1";
using var store = GetDocumentStore();
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from Commands where Error = null"
};
var subsName = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
await using (var acceptedSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsName)))
{
var isProcessed = new AsyncManualResetEvent();
var task = acceptedSubscription.Run(x =>
{
foreach (var item in x.Items)
{
if (item.Id == id)
{
isProcessed.Set();
}
}
});
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new Command(), id);
await session.SaveChangesAsync();
}
Assert.True(await isProcessed.WaitAsync(TimeSpan.FromSeconds(15)));
}
}
public void Subscribe()
{
const string subName = "NewOrders";
var subscriptionCreationOptions = new SubscriptionCreationOptions
{
Name = subName,
};
try
{
InternetShopStore.Store.Subscriptions.GetSubscriptionState(subName, InternetShopStore.DatabaseName);
}
catch (SubscriptionDoesNotExistException)
{
InternetShopStore.Store.Subscriptions.Create <Order>(options: subscriptionCreationOptions, database: InternetShopStore.DatabaseName);
}
_subscription = InternetShopStore.Store.Subscriptions.GetSubscriptionWorker <Order>(subName);
_subscription.Run(x =>
x.Items.ForEach(async item =>
{
await _hubContext.Clients.All.SendAsync("OrderAdded", $"{item.Result.Product.Name}");
}),
CancellationToken.None);
}
public async Task SubscriptionStrategyConnectIfFree()
{
using (var store = GetDocumentStore())
{
await CreateDocuments(store, 1);
var lastChangeVector = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector ?? null;
await CreateDocuments(store, 5);
var subscriptionCreationParams = new SubscriptionCreationOptions()
{
Query = "from Things",
ChangeVector = lastChangeVector
};
var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
using (
var acceptedSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(20)
}))
{
var acceptedSubscriptionList = new BlockingCollection <Thing>();
var firstSubscriptionTask = acceptedSubscription.Run(u =>
{
foreach (var item in u.Items)
{
acceptedSubscriptionList.Add(item.Result);
}
});
GC.KeepAlive(firstSubscriptionTask);
Thing thing;
// wait until we know that connection was established
for (var i = 0; i < 5; i++)
{
Assert.True(acceptedSubscriptionList.TryTake(out thing, 1000));
}
Assert.False(acceptedSubscriptionList.TryTake(out thing, 50));
// open second subscription
using (var rejectedSubscription =
store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId)
{
Strategy = SubscriptionOpeningStrategy.OpenIfFree,
TimeToWaitBeforeConnectionRetry = TimeSpan.FromMilliseconds(2000)
}))
{
// sometime not throwing (on linux) when written like this:
// await Assert.ThrowsAsync<SubscriptionInUseException>(async () => await rejectedSubscription.StartAsync());
// so we put this in a try block
try
{
await rejectedSubscription.Run(_ => { });
Assert.False(true, "Exepcted a throw here");
}
catch (SubscriptionInUseException)
{
}
}
}
}
}
public async Task SubscriptionSimpleTakeOverStrategy()
{
using (var store = GetDocumentStore())
{
await CreateDocuments(store, 1);
var lastChangeVector = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector ?? null;
await CreateDocuments(store, 5);
var subscriptionCreationParams = new SubscriptionCreationOptions()
{
Query = "from Things",
ChangeVector = lastChangeVector
};
var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
using (
var acceptedSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId)))
{
var acceptedSusbscriptionList = new BlockingCollection <Thing>();
var takingOverSubscriptionList = new BlockingCollection <Thing>();
long counter = 0;
var batchProccessedByFirstSubscription = new AsyncManualResetEvent();
acceptedSubscription.AfterAcknowledgment +=
b =>
{
if (Interlocked.Read(ref counter) == 5)
{
batchProccessedByFirstSubscription.Set();
}
return(Task.CompletedTask);
};
var firstRun = acceptedSubscription.Run(x =>
{
foreach (var item in x.Items)
{
Interlocked.Increment(ref counter);
acceptedSusbscriptionList.Add(item.Result);
}
});
Thing thing;
// wait until we know that connection was established
for (var i = 0; i < 5; i++)
{
Assert.True(acceptedSusbscriptionList.TryTake(out thing, 5000), "no doc");
}
Assert.True(await batchProccessedByFirstSubscription.WaitAsync(TimeSpan.FromSeconds(15)), "no ack");
Assert.False(acceptedSusbscriptionList.TryTake(out thing));
// open second subscription
using (var takingOverSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(
new SubscriptionWorkerOptions(subsId)
{
Strategy = SubscriptionOpeningStrategy.TakeOver
}))
{
GC.KeepAlive(takingOverSubscription.Run(x =>
{
foreach (var item in x.Items)
{
takingOverSubscriptionList.Add(item.Result);
}
}));
Assert.ThrowsAsync <SubscriptionInUseException>(() => firstRun).Wait();
await CreateDocuments(store, 5);
// wait until we know that connection was established
for (var i = 0; i < 5; i++)
{
Assert.True(takingOverSubscriptionList.TryTake(out thing, 5000), "no doc takeover");
}
Assert.False(takingOverSubscriptionList.TryTake(out thing));
}
}
}
}
public async Task SubscriptionWaitStrategy()
{
using (var store = GetDocumentStore())
{
await CreateDocuments(store, 1);
var lastChangeVector = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector;
var subscriptionCreationParams = new SubscriptionCreationOptions()
{
Query = "from Things",
ChangeVector = lastChangeVector
};
await CreateDocuments(store, 5);
var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
using (
var acceptedSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId)))
{
var acceptedSusbscriptionList = new BlockingCollection <Thing>();
var waitingSubscriptionList = new BlockingCollection <Thing>();
var ackSentAmre = new AsyncManualResetEvent();
acceptedSubscription.AfterAcknowledgment += b => { ackSentAmre.Set(); return(Task.CompletedTask); };
GC.KeepAlive(acceptedSubscription.Run(x =>
{
foreach (var item in x.Items)
{
acceptedSusbscriptionList.Add(item.Result);
}
Thread.Sleep(20);
}));
// wait until we know that connection was established
Thing thing;
// wait until we know that connection was established
for (var i = 0; i < 5; i++)
{
Assert.True(acceptedSusbscriptionList.TryTake(out thing, 50000));
}
Assert.False(acceptedSusbscriptionList.TryTake(out thing, 50));
// open second subscription
using (
var waitingSubscription =
store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId)
{
Strategy = SubscriptionOpeningStrategy.WaitForFree,
TimeToWaitBeforeConnectionRetry = TimeSpan.FromMilliseconds(250)
}))
{
GC.KeepAlive(waitingSubscription.Run(x =>
{
foreach (var item in x.Items)
{
waitingSubscriptionList.Add(item.Result);
}
}));
Assert.True(await ackSentAmre.WaitAsync(TimeSpan.FromSeconds(50)));
acceptedSubscription.Dispose();
await CreateDocuments(store, 5);
// wait until we know that connection was established
for (var i = 0; i < 5; i++)
{
Assert.True(waitingSubscriptionList.TryTake(out thing, 3000));
}
Assert.False(waitingSubscriptionList.TryTake(out thing, 50));
}
}
}
}
public async Task CriteriaScriptWithTransformation(bool useSsl)
{
string dbName = GetDatabaseName();
X509Certificate2 clientCertificate = null;
X509Certificate2 adminCertificate = null;
if (useSsl)
{
var serverCertPath = SetupServerAuthentication();
adminCertificate = AskServerForClientCertificate(serverCertPath, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin);
clientCertificate = AskServerForClientCertificate(serverCertPath, new Dictionary <string, DatabaseAccess>
{
[dbName] = DatabaseAccess.ReadWrite,
});
}
using (var store = GetDocumentStore(new Options
{
AdminCertificate = adminCertificate,
ClientCertificate = clientCertificate,
ModifyDatabaseName = s => dbName
}))
{
using (var subscriptionManager = new DocumentSubscriptions(store))
{
await CreateDocuments(store, 1);
var lastChangeVector = (await store.Admin.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector;
await CreateDocuments(store, 6);
var subscriptionCreationParams = new SubscriptionCreationOptions()
{
Query = @"
declare function project(d) {
var namSuffix = parseInt(d.Name.replace('ThingNo', ''));
if (namSuffix <= 2){
return false;
}
else if (namSuffix == 3){
return null;
}
else if (namSuffix == 4){
return d;
}
return {Name: 'foo', OtherDoc:load('things/6-A')}
}
from Things as d
select project(d)
",
ChangeVector = lastChangeVector
};
var subsId = subscriptionManager.Create(subscriptionCreationParams);
using (var subscription = subscriptionManager.Open <BlittableJsonReaderObject>(new SubscriptionConnectionOptions(subsId)))
{
using (store.GetRequestExecutor().ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var list = new BlockingCollection <BlittableJsonReaderObject>();
GC.KeepAlive(subscription.Run(x =>
{
foreach (var item in x.Items)
{
list.Add(context.ReadObject(item.Result, "test"));
}
}));
BlittableJsonReaderObject thing;
Assert.True(list.TryTake(out thing, 5000));
dynamic dynamicThing = new DynamicBlittableJson(thing);
Assert.Equal("ThingNo4", dynamicThing.Name);
Assert.True(list.TryTake(out thing, 5000));
dynamicThing = new DynamicBlittableJson(thing);
Assert.Equal("foo", dynamicThing.Name);
Assert.Equal("ThingNo4", dynamicThing.OtherDoc.Name);
Assert.False(list.TryTake(out thing, 50));
}
}
}
}
}
private async Task <RunResult> SingleTestRun(int workers, int fakeProcessingTimePerBatch, bool script, bool revision = false)
{
try
{
if (script == false && revision)
{
throw new InvalidOperationException("Can't have a revision without script");
}
string revisions = revision ? " (Revisions = true)" : "";
if (script)
{
SubscriptionCreationOptions subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from " + _collectionName + revisions
};
_subscriptionName = await _store.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false);
Console.WriteLine($"Created Subscription with query: '{subscriptionCreationParams.Query}'");
}
else
{
_subscriptionName = await _store.Subscriptions.CreateAsync <Order>();
Console.WriteLine($"Created Subscription without query");
}
EnableRevisions();
var workersList = new List <SubscriptionWorker <dynamic> >();
var taskList = new List <Task>();
Stopwatch sp = null;
for (int i = 0; i < workers; i++)
{
var worker = _store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(_subscriptionName)
{
Strategy = SubscriptionOpeningStrategy.Concurrent,
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5),
MaxDocsPerBatch = _batchSize
});
workersList.Add(worker);
worker.OnEstablishedSubscriptionConnection += () =>
{
Interlocked.CompareExchange(ref sp, Stopwatch.StartNew(), null);
};
}
var tcs = new TaskCompletionSource <RunResult>();
var sentCount = 0;
foreach (var worker in workersList)
{
taskList.Add(worker.Run((async o =>
{
if (Interlocked.Add(ref sentCount, o.Items.Count) >= _docsAmountToTest)
{
tcs.TrySetResult(new RunResult
{
DocsProccessed = sentCount,
DocsRequested = _batchSize,
ElapsedMs = sp.ElapsedMilliseconds
});
}
await Task.Delay(fakeProcessingTimePerBatch);
})));
}
var result = await tcs.Task.ConfigureAwait(false);
foreach (var worker in workersList)
{
await worker.DisposeAsync().ConfigureAwait(false);
}
await Task.WhenAll(taskList);
return(result);
}
catch (Exception ex)
{
Console.WriteLine(ex);
throw;
}
}
public CreateSubscriptionCommand(SubscriptionCreationOptions options, string id = null)
{
_options = options;
_id = id;
}
public async Task CanDisableTcpCompressionOnTheClientViaStoreConventions()
{
var server = GetNewServer(new ServerCreationOptions
{
RunInMemory = false
});
using (var store1 = GetDocumentStore(new Options
{
Server = server
}))
using (var store2 = GetDocumentStore(new Options
{
Server = server,
ModifyDocumentStore = s => s.Conventions.DisableTcpCompression = true
}))
{
Assert.False(server.Configuration.Server.DisableTcpCompression);
foreach (var store in new [] { store1, store2 })
{
var compressionDisabled = store == store2;
using (var session = store.OpenSession())
{
for (int i = 0; i < 10; i++)
{
session.Store(new User());
}
session.SaveChanges();
}
var subscriptionCreationParams = new SubscriptionCreationOptions()
{
Query = "from Users"
};
var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
using (var subscription = store.Subscriptions.GetSubscriptionWorker <User>(new SubscriptionWorkerOptions(subsId)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var list = new BlockingCollection <User>();
GC.KeepAlive(subscription.Run(u =>
{
foreach (var item in u.Items)
{
list.Add(item.Result);
}
}));
User user;
for (var i = 0; i < 10; i++)
{
Assert.True(list.TryTake(out user, 1000));
}
Assert.False(list.TryTake(out user, 50));
var db = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var tcpConnections = db.RunningTcpConnections.ToList();
Assert.Equal(1, tcpConnections.Count);
if (compressionDisabled)
{
Assert.False(tcpConnections[0].Stream is Sparrow.Utils.ReadWriteCompressedStream);
}
else
{
Assert.True(tcpConnections[0].Stream is Sparrow.Utils.ReadWriteCompressedStream);
}
}
}
}
}
public CreateSubscriptionCommand(SubscriptionCreationOptions options)
{
_options = options;
}
public async Task SubscribtionWithEtag()
{
using (var store = GetDocumentStore())
{
var us1 = new User {
Id = "users/1", Name = "john", Age = 22
};
var us2 = new User {
Id = "users/2", Name = "KY", Age = 30
};
var us3 = new User {
Id = "users/3", Name = "BEN", Age = 30
};
var us4 = new User {
Id = "users/4", Name = "Hila", Age = 29
};
var us5 = new User {
Id = "users/5", Name = "Revital", Age = 34
};
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(us1);
await session.StoreAsync(us2);
await session.StoreAsync(us3);
await session.StoreAsync(us4);
await session.StoreAsync(us5);
await session.SaveChangesAsync();
var user2ChangeVector = session.Advanced.GetChangeVectorFor(us2);
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from Users",
ChangeVector = user2ChangeVector
};
var id = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
var users = new List <dynamic>();
using (var subscription = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(id)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var docs = new BlockingCollection <dynamic>();
var keys = new BlockingCollection <string>();
var ages = new BlockingCollection <int>();
GC.KeepAlive(subscription.Run(u =>
{
foreach (var x in u.Items)
{
keys.Add(x.Result.Id);
ages.Add(x.Result.Age);
docs.Add((x.Result));
}
}));
dynamic doc;
Assert.True(docs.TryTake(out doc, _waitForDocTimeout));
users.Add(doc);
Assert.True(docs.TryTake(out doc, _waitForDocTimeout));
users.Add(doc);
Assert.True(docs.TryTake(out doc, _waitForDocTimeout));
users.Add(doc);
var cnt = users.Count;
Assert.Equal(3, cnt);
string key;
Assert.True(keys.TryTake(out key, _waitForDocTimeout));
Assert.Equal("users/3", key);
Assert.True(keys.TryTake(out key, _waitForDocTimeout));
Assert.Equal("users/4", key);
Assert.True(keys.TryTake(out key, _waitForDocTimeout));
Assert.Equal("users/5", key);
int age;
Assert.True(ages.TryTake(out age, _waitForDocTimeout));
Assert.Equal(30, age);
Assert.True(ages.TryTake(out age, _waitForDocTimeout));
Assert.Equal(29, age);
Assert.True(ages.TryTake(out age, _waitForDocTimeout));
Assert.Equal(34, age);
}
}
}
}
public async Task CanDisableTcpCompressionViaConfiguration_SubscriptionsTest()
{
var server = GetNewServer(new ServerCreationOptions
{
RunInMemory = false,
CustomSettings = new Dictionary <string, string>()
{
[RavenConfiguration.GetKey(x => x.Server.DisableTcpCompression)] = "true"
}
});
Assert.True(server.Configuration.Server.DisableTcpCompression);
using (var store = GetDocumentStore(new Options
{
Server = server
}))
{
using (var session = store.OpenSession())
{
for (int i = 0; i < 10; i++)
{
session.Store(new User());
}
session.SaveChanges();
}
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from Users"
};
var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
using (var subscription = store.Subscriptions.GetSubscriptionWorker <User>(new SubscriptionWorkerOptions(subsId)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var list = new BlockingCollection <User>();
GC.KeepAlive(subscription.Run(u =>
{
foreach (var item in u.Items)
{
list.Add(item.Result);
}
}));
User user;
for (var i = 0; i < 10; i++)
{
Assert.True(list.TryTake(out user, 1000));
}
Assert.False(list.TryTake(out user, 50));
var db = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var tcpConnections = db.RunningTcpConnections.ToList();
Assert.Equal(1, tcpConnections.Count);
// assert non-compressed tcp connection
Assert.False(tcpConnections[0].Stream is Sparrow.Utils.ReadWriteCompressedStream);
}
}
}
public override void RunActualTest()
{
using (DocumentStore.Initialize())
{
ReportInfo("Inserting products docs");
InsertProducts();
ReportInfo("Inserting shippers docs");
InsertShippers();
ReportInfo("Bulk insert users docs");
var bulkInsertUsersTask = Task.Run(() => BulkInsertUsersDocuments());
try
{
var usersSubscription = new SubscriptionCreationOptions <User2>
{
Name = $"UsersSubscription.{GenralGuid}",
Filter = x => (x.Age % 2) == 0
};
var orderSubscription = new SubscriptionCreationOptions <User2>
{
Name = $"CreateOrderSubscription.{GenralGuid}",
Filter = x => (x.Products != null),
Projection = x => new
{
ProductsNames = x.Products
}
};
ReportInfo("Create subscriptions : UsersSubscription");
var createUsersSubscription = DocumentStore.Subscriptions.Create(usersSubscription);
ReportInfo("Create subscriptions : CreateOrderSubscription");
var createOrderSubscription = DocumentStore.Subscriptions.Create(orderSubscription);
var usersSubscriptionWorkerOptions = new SubscriptionWorkerOptions(createUsersSubscription)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5),
MaxDocsPerBatch = 20,
CloseWhenNoDocsLeft = false
};
var orderSubscriptionWorkerOptions = new SubscriptionWorkerOptions(createOrderSubscription)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5),
MaxDocsPerBatch = 3,
CloseWhenNoDocsLeft = false
};
ReportInfo("UsersSubscription: get subscriptions worker");
var usersSubscriptionWorker = DocumentStore.Subscriptions.GetSubscriptionWorker <dynamic>(usersSubscriptionWorkerOptions);
ReportInfo("CreateOrderSubscription: get subscriptions worker");
var orderSubscriptionWorker = DocumentStore.Subscriptions.GetSubscriptionWorker <dynamic>(orderSubscriptionWorkerOptions);
ReportInfo("Start inserting products to users");
var usersSubscriptionRun = Task.Run(() => InsertProductsToUsers(usersSubscriptionWorker));
ReportInfo("Start creating orders");
var orderSubscriptionRun = Task.Run(() => CreateOrderDoc(orderSubscriptionWorker));
var i = 0;
foreach (var shipper in _shipperGuid)
{
var s = "shipper." + GenralGuid + "/" + shipper;
var shipperSubscription = new SubscriptionCreationOptions <Order>
{
Name = $"shipperSubscription-{i}.{GenralGuid}",
Filter = x => (x.ShipVia.StartsWith(s))
};
ReportInfo($"Create subscriptions : shipperSubscription-{i}.{GenralGuid}");
var createShipperSubscription = DocumentStore.Subscriptions.Create(shipperSubscription);
var shipperSubscriptionWorkerOptions = new SubscriptionWorkerOptions(createShipperSubscription)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5),
MaxDocsPerBatch = 6,
CloseWhenNoDocsLeft = false
};
ReportInfo($"shipperSubscription-{i}.{GenralGuid}: get subscriptions worker");
var shipperSubscriptionWorker = DocumentStore.Subscriptions.GetSubscriptionWorker <Order>(shipperSubscriptionWorkerOptions);
var i1 = i;
ReportInfo($"Start counting orders for shipper {i}. Id: {_shipperGuid[i]}");
var getShipperTask = Task.Run(() => GetShipper(shipperSubscriptionWorker, i1));
_tasks.AddLast(getShipperTask);
i += 1;
}
}
catch (Exception e)
{
ReportFailure("Error:", e);
}
while (_tasks.All(x => (x.IsCompleted)) == false)
{
}
var success = true;
for (int i = 0; i < ShippersCount; i++)
{
if (_shipper[i] != _shipperRes[i])
{
ReportInfo($"{i}: {_shipper[i]} != {_shipperRes[i]}");
success = false;
break;
}
}
if (success)
{
ReportSuccess("Test done");
}
else
{
ReportFailure("Test Failed", null);
}
}
}
public async Task SubscribtionWithEtag_MultipleOpens()
{
using (var store = GetDocumentStore())
{
var us1 = new User {
Id = "users/1", Name = "john", Age = 22
};
var us2 = new User {
Id = "users/2", Name = "KY", Age = 30
};
var us3 = new User {
Id = "users/3", Name = "BEN", Age = 30
};
var us4 = new User {
Id = "users/4", Name = "Hila", Age = 29
};
var us5 = new User {
Id = "users/5", Name = "Revital", Age = 34
};
string subscriptionName;
var subscriptionReleasedAwaiter = Task.CompletedTask;
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(us1);
await session.StoreAsync(us2);
await session.StoreAsync(us3);
await session.StoreAsync(us4);
await session.StoreAsync(us5);
await session.SaveChangesAsync();
var user2ChangeVector = session.Advanced.GetChangeVectorFor(us2);
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from Users",
ChangeVector = user2ChangeVector
};
subscriptionName = await store.Subscriptions.CreateAsync(subscriptionCreationParams);
var users = new List <dynamic>();
using (var subscription = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(subscriptionName)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var docs = new BlockingCollection <dynamic>();
var keys = new BlockingCollection <string>();
var ages = new BlockingCollection <int>();
var mre = new ManualResetEvent(false);
subscription.AfterAcknowledgment += batch =>
{
mre.Set();
return(Task.CompletedTask);
};
GC.KeepAlive(subscription.Run(u =>
{
foreach (var x in u.Items)
{
keys.Add(x.Result.Id);
ages.Add(x.Result.Age);
docs.Add((x.Result));
}
}));
var db = await GetDatabase(store.Database);
var subscriptionState = db.SubscriptionStorage.GetSubscriptionFromServerStore(subscriptionName);
subscriptionReleasedAwaiter = db.SubscriptionStorage.GetSubscriptionConnectionInUseAwaiter(subscriptionState.SubscriptionId);
dynamic doc;
Assert.True(docs.TryTake(out doc, _waitForDocTimeout));
users.Add(doc);
Assert.True(docs.TryTake(out doc, _waitForDocTimeout));
users.Add(doc);
Assert.True(docs.TryTake(out doc, _waitForDocTimeout));
users.Add(doc);
var cnt = users.Count;
Assert.Equal(3, cnt);
string key;
Assert.True(keys.TryTake(out key, _waitForDocTimeout));
Assert.Equal("users/3", key);
Assert.True(keys.TryTake(out key, _waitForDocTimeout));
Assert.Equal("users/4", key);
Assert.True(keys.TryTake(out key, _waitForDocTimeout));
Assert.Equal("users/5", key);
int age;
Assert.True(ages.TryTake(out age, _waitForDocTimeout));
Assert.Equal(30, age);
Assert.True(ages.TryTake(out age, _waitForDocTimeout));
Assert.Equal(29, age);
Assert.True(ages.TryTake(out age, _waitForDocTimeout));
Assert.Equal(34, age);
Assert.True(mre.WaitOne(250));
}
}
Assert.True(Task.WaitAll(new[] { subscriptionReleasedAwaiter }, 250));
using (var subscription = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(subscriptionName)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var docs = new BlockingCollection <dynamic>();
GC.KeepAlive(subscription.Run(x =>
{
docs.Add((x));
}));
dynamic item;
var tryTake = docs.TryTake(out item, TimeSpan.FromMilliseconds(250));
Assert.False(tryTake);
}
}
}
public async Task SubscriptionShouldRespectDocumentsWithCompressedData()
{
using (var documentStore = this.GetDocumentStore())
{
Server.ServerStore.Observer.Suspended = true;
var originalDoc = new Doc
{
Id = "doc/1",
StrVal = new string(Enumerable.Repeat('.', 129).ToArray()),
LongByteArray = Enumerable.Repeat((byte)2, 1024).ToArray()
};
using (var session = documentStore.OpenAsyncSession())
{
await session.StoreAsync(originalDoc);
await session.SaveChangesAsync();
}
var database = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(documentStore.Database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var doc = database.DocumentsStorage.Get(context, "doc/1");
MemoryStream ms = new MemoryStream();
using (var newContext = JsonOperationContext.ShortTermSingleUse())
using (var writer = new BlittableJsonTextWriter(newContext, ms))
{
writer.WriteDocument(newContext, doc, metadataOnly: false);
writer.Flush();
var bjro = GetReaderFromMemoryStream(ms, context);
var desereializedDoc = (Doc)EntityToBlittable.ConvertToEntity(typeof(Doc), null, bjro, DocumentConventions.Default);
Assert.Equal(originalDoc.StrVal, desereializedDoc.StrVal);
Assert.Equal(originalDoc.LongByteArray, originalDoc.LongByteArray);
}
}
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from Docs",
};
var subsId = await documentStore.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false);
var amre = new AsyncManualResetEvent();
using (var subscription = documentStore.Subscriptions.GetSubscriptionWorker <Doc>(new SubscriptionWorkerOptions(subsId)))
{
var t = subscription.Run(batch =>
{
var receivedDoc = batch.Items.First().Result;
Assert.Equal(originalDoc.LongByteArray, receivedDoc.LongByteArray);
Assert.Equal(originalDoc.StrVal, receivedDoc.StrVal);
amre.Set();
});
try
{
Assert.True(await amre.WaitAsync(TimeSpan.FromSeconds(60)));
}
catch
{
if (t.IsFaulted)
{
t.Wait();
}
throw;
}
}
}
}
public CreateSubscriptionCommand(DocumentConventions conventions, SubscriptionCreationOptions options, string id = null)
{
_conventions = conventions;
_options = options;
_id = id;
}
public CreateSubscriptionCommand(DocumentConventions conventions, SubscriptionCreationOptions options, string id = null)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_id = id;
}
