public void CanNameAndOpenWithNameOnly() { using (var store = GetDocumentStore()) { var subscriptionCreationParams = new SubscriptionCreationOptions { Name = "get-users", }; var subscriptionId = store.Subscriptions.Create <User>(options: subscriptionCreationParams); var subscription = store.Subscriptions.Open <User>(new SubscriptionConnectionOptions("get-users")); var users = new BlockingCollection <User>(); using (var session = store.OpenSession()) { session.Store(new User()); session.SaveChanges(); } var subscirptionLifetimeTask = subscription.Run(u => { foreach (var item in u.Items) { users.Add(item.Result); } }); User user; Assert.True(users.TryTake(out user, 1000000)); } }
public void GetSubscriptionTaskInfo() { var subscriptionOption = new SubscriptionCreationOptions <Query.Order> { Name = "sub", Filter = x => x.Employee.StartsWith("e"), MentorNode = "B" }; using (var store = GetDocumentStore()) { var sub = store.Subscriptions.Create(subscriptionOption); var op = new GetOngoingTaskInfoOperation(subscriptionOption.Name, OngoingTaskType.Subscription); var subscriptionResult = (OngoingTaskSubscription)store.Maintenance.Send(op); Assert.Equal(subscriptionOption.MentorNode, subscriptionResult.MentorNode); Assert.NotNull(subscriptionResult.Query); var state = store.Subscriptions.GetSubscriptionState(subscriptionOption.Name); op = new GetOngoingTaskInfoOperation(state.SubscriptionId, OngoingTaskType.Subscription); subscriptionResult = (OngoingTaskSubscription)store.Maintenance.Send(op); Assert.Equal(subscriptionOption.MentorNode, subscriptionResult.MentorNode); Assert.Equal(subscriptionOption.Name, subscriptionResult.SubscriptionName); Assert.NotNull(subscriptionResult.Query); op = new GetOngoingTaskInfoOperation(state.SubscriptionId - 1, OngoingTaskType.Subscription); Assert.Throws <SubscriptionDoesNotExistException>(() => store.Maintenance.Send(op)); } }
public async Task BasicSusbscriptionTest() { using (var store = GetDocumentStore()) { await CreateDocuments(store, 1); var lastChangeVector = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector; await CreateDocuments(store, 5); var subscriptionCreationParams = new SubscriptionCreationOptions() { Query = "from Things", ChangeVector = lastChangeVector }; var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams); using (var subscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId))) { var list = new BlockingCollection <Thing>(); GC.KeepAlive(subscription.Run(u => { foreach (var item in u.Items) { list.Add(item.Result); } })); Thing thing; for (var i = 0; i < 5; i++) { Assert.True(list.TryTake(out thing, 1000)); } Assert.False(list.TryTake(out thing, 50)); } } }
public async Task <long> PutSubscription(SubscriptionCreationOptions options, string raftRequestId, long?subscriptionId = null, bool?disabled = false, string mentor = null) { var command = new PutSubscriptionCommand(_db.Name, options.Query, mentor, raftRequestId) { InitialChangeVector = options.ChangeVector, SubscriptionName = options.Name, SubscriptionId = subscriptionId, Disabled = disabled ?? false }; var(etag, _) = await _serverStore.SendToLeaderAsync(command); if (_logger.IsInfoEnabled) { _logger.Info($"New Subscription with index {etag} was created"); } await _db.RachisLogIndexNotifications.WaitForIndexNotification(etag, _serverStore.Engine.OperationTimeout); if (subscriptionId != null) { // updated existing subscription return(subscriptionId.Value); } return(etag); }
public async Task BasicCriteriaTest(bool useSsl) { string dbName = GetDatabaseName(); X509Certificate2 clientCertificate = null; X509Certificate2 adminCertificate = null; if (useSsl) { var certificates = Certificates.SetupServerAuthentication(); adminCertificate = Certificates.RegisterClientCertificate(certificates.ServerCertificate.Value, certificates.ClientCertificate1.Value, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin); clientCertificate = Certificates.RegisterClientCertificate(certificates.ServerCertificate.Value, certificates.ClientCertificate2.Value, new Dictionary <string, DatabaseAccess> { [dbName] = DatabaseAccess.ReadWrite }); } using (var store = GetDocumentStore(new Options { AdminCertificate = adminCertificate, ClientCertificate = clientCertificate, ModifyDatabaseName = s => dbName })) { using (var subscriptionManager = new DocumentSubscriptions(store)) { await CreateDocuments(store, 1); var lastChangeVector = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector; await CreateDocuments(store, 5); var subscriptionCreationParams = new SubscriptionCreationOptions() { Query = "from Things where Name = 'ThingNo3'", ChangeVector = lastChangeVector }; var subsId = subscriptionManager.Create(subscriptionCreationParams); using (var subscription = subscriptionManager.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { var list = new BlockingCollection <Thing>(); GC.KeepAlive(subscription.Run(x => { foreach (var item in x.Items) { list.Add(item.Result); } })); Thing thing; Assert.True(list.TryTake(out thing, _reasonableWaitTime)); Assert.Equal("ThingNo3", thing.Name); Assert.False(list.TryTake(out thing, 50)); } } } }
private static bool SubscriptionHasChanges(SubscriptionCreationOptions options, SubscriptionState state) { bool gotChanges = options.Name != state.SubscriptionName || options.ChangeVector != state.ChangeVectorForNextBatchStartingPoint || options.MentorNode != state.MentorNode || options.Query != state.Query; return(gotChanges); }
public async Task BasicSusbscriptionTest() { using (var store = GetDocumentStore()) { await CreateDocuments(store, 1); var lastChangeVector = store.Maintenance.Send(new GetStatisticsOperation()).DatabaseChangeVector; await CreateDocuments(store, 5); var subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from Things", ChangeVector = lastChangeVector }; var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false); using (var subscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { var bc = new BlockingCollection <Thing>(); GC.KeepAlive(subscription.Run(x => { foreach (var item in x.Items) { bc.Add(item.Result); } })); Thing thing; for (var i = 0; i < 5; i++) { Assert.True(bc.TryTake(out thing, 1000)); } Assert.False(bc.TryTake(out thing, 50)); for (var j = 0; j < 2; j++) { await CreateDocuments(store, 1); Assert.True(bc.TryTake(out thing, 500)); Assert.False(bc.TryTake(out thing, 50)); } } } }
private async Task <RunResult> SingleTestRun() { try { if (string.IsNullOrEmpty(_subscriptionName)) { var subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from " + _collectionName }; _subscriptionName = await _store.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false); } using (var subscription = _store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(_subscriptionName) { Strategy = SubscriptionOpeningStrategy.WaitForFree, TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { var tcs = new TaskCompletionSource <object>(); var sp = Stopwatch.StartNew(); int count = 0; var task = subscription.Run(o => { if (count++ >= _batchSize) { tcs.TrySetResult(null); } });; await tcs.Task.ConfigureAwait(false); await subscription.DisposeAsync().ConfigureAwait(false); await task; return(new RunResult { DocsProccessed = count, DocsRequested = _batchSize, ElapsedMs = sp.ElapsedMilliseconds }); } } catch (Exception ex) { Console.WriteLine(ex); throw; } }
public async Task ShouldRespectCollectionCriteria() { using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { for (int i = 0; i < 100; i++) { await session.StoreAsync(new Company()); await session.StoreAsync(new User()); await session.StoreAsync(new Address()); } await session.SaveChangesAsync(); } var subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from Users" }; var id = await store.Subscriptions.CreateAsync(subscriptionCreationParams); using (var subscription = store.Subscriptions.GetSubscriptionWorker( new SubscriptionWorkerOptions(id) { MaxDocsPerBatch = 31, TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { var ids = new List <string>(); GC.KeepAlive(subscription.Run(batch => { foreach (var item in batch.Items) { ids.Add(item.Id); } })); Assert.True(SpinWait.SpinUntil(() => ids.Count >= 100, TimeSpan.FromSeconds(60))); Assert.Equal(100, ids.Count); foreach (var i in ids) { Assert.True(i.StartsWith("users/")); } } } }
private async Task CreateInternal(BlittableJsonReaderObject bjro, SubscriptionCreationOptions options, DocumentsOperationContext context, long?id, bool?disabled) { if (TrafficWatchManager.HasRegisteredClients) { AddStringToHttpContext(bjro.ToString(), TrafficWatchChangeType.Subscriptions); } var sub = SubscriptionConnection.ParseSubscriptionQuery(options.Query); if (Enum.TryParse(options.ChangeVector, out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue)) { switch (changeVectorSpecialValue) { case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime: options.ChangeVector = null; break; case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument: options.ChangeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, sub.Collection); break; } } var mentor = options.MentorNode; var subscriptionId = await Database.SubscriptionStorage.PutSubscription(options, GetRaftRequestIdFromQuery(), id, disabled, mentor); var name = options.Name ?? subscriptionId.ToString(); using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext)) using (serverContext.OpenReadTransaction()) { // need to wait on the relevant remote node var node = Database.SubscriptionStorage.GetResponsibleNode(serverContext, name); if (node != null && node != ServerStore.NodeTag) { await WaitForExecutionOnSpecificNode(serverContext, ServerStore.GetClusterTopology(serverContext), node, subscriptionId); } } HttpContext.Response.StatusCode = (int)HttpStatusCode.Created; await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(CreateSubscriptionResult.Name)] = name }); } }
public void ShouldStopPullingTaskWhenSubscriptionIsDeleted() { using (var store = GetDocumentStore()) { // insert few documents and fetch them using subscription using (var session = store.OpenSession()) { for (int i = 0; i < 10; i++) { session.Store(new Company()); } session.SaveChanges(); } var subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from Companies" }; var id = store.Subscriptions.Create(subscriptionCreationParams); var subscription = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(id) { MaxDocsPerBatch = 5, TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) }); var docs = new List <dynamic>(); GC.KeepAlive(subscription.Run(batch => docs.AddRange(batch.Items.Select(item => item.Result)))); Assert.True(SpinWait.SpinUntil(() => docs.Count == 10, TimeSpan.FromSeconds(60))); // all documents were fetched - time to delete subscription store.Subscriptions.DeleteAsync(id).Wait(); // verify if we don't get new items using (var session = store.OpenSession()) { for (int i = 0; i < 2; i++) { session.Store(new Company()); } session.SaveChanges(); } // wait 3 seconds for new documents - we shouldn't get any Assert.False(SpinWait.SpinUntil(() => docs.Count != 10, TimeSpan.FromSeconds(3))); } }
public async Task <long> PutSubscription(SubscriptionCreationOptions options, long?subscriptionId = null, bool?disabled = false) { var command = new PutSubscriptionCommand(_db.Name, options.Query) { InitialChangeVector = options.ChangeVector, SubscriptionName = options.Name, SubscriptionId = subscriptionId, Disabled = disabled ?? false }; var(etag, _) = await _serverStore.SendToLeaderAsync(command); if (_logger.IsInfoEnabled) { _logger.Info($"New Subscription with index {etag} was created"); } await _db.RachisLogIndexNotifications.WaitForIndexNotification(etag); return(etag); }
public async Task CreateSubscription() { using (var store = GetDocumentStore()) { var subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from People" }; var subsName = await store.Subscriptions.CreateAsync(subscriptionCreationParams); var subscriptionsConfig = await store.Subscriptions.GetSubscriptionsAsync(0, 10); var subscripitonState = await store.Subscriptions.GetSubscriptionStateAsync(subsName); Assert.Equal(1, subscriptionsConfig.Count); Assert.Equal(subscriptionCreationParams.Query, subscriptionsConfig[0].Query); Assert.Null(subscriptionsConfig[0].ChangeVectorForNextBatchStartingPoint); Assert.Equal(subsName, subscriptionsConfig[0].SubscriptionName); Assert.Equal(subscripitonState.SubscriptionId, subscriptionsConfig[0].SubscriptionId); } }
public async Task Subscription_WhenFilteredByNull_ShouldWork() { const string id = "A1"; using var store = GetDocumentStore(); var subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from Commands where Error = null" }; var subsName = await store.Subscriptions.CreateAsync(subscriptionCreationParams); await using (var acceptedSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsName))) { var isProcessed = new AsyncManualResetEvent(); var task = acceptedSubscription.Run(x => { foreach (var item in x.Items) { if (item.Id == id) { isProcessed.Set(); } } }); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new Command(), id); await session.SaveChangesAsync(); } Assert.True(await isProcessed.WaitAsync(TimeSpan.FromSeconds(15))); } }
public void Subscribe() { const string subName = "NewOrders"; var subscriptionCreationOptions = new SubscriptionCreationOptions { Name = subName, }; try { InternetShopStore.Store.Subscriptions.GetSubscriptionState(subName, InternetShopStore.DatabaseName); } catch (SubscriptionDoesNotExistException) { InternetShopStore.Store.Subscriptions.Create <Order>(options: subscriptionCreationOptions, database: InternetShopStore.DatabaseName); } _subscription = InternetShopStore.Store.Subscriptions.GetSubscriptionWorker <Order>(subName); _subscription.Run(x => x.Items.ForEach(async item => { await _hubContext.Clients.All.SendAsync("OrderAdded", $"{item.Result.Product.Name}"); }), CancellationToken.None); }
public async Task SubscriptionStrategyConnectIfFree() { using (var store = GetDocumentStore()) { await CreateDocuments(store, 1); var lastChangeVector = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector ?? null; await CreateDocuments(store, 5); var subscriptionCreationParams = new SubscriptionCreationOptions() { Query = "from Things", ChangeVector = lastChangeVector }; var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams); using ( var acceptedSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(20) })) { var acceptedSubscriptionList = new BlockingCollection <Thing>(); var firstSubscriptionTask = acceptedSubscription.Run(u => { foreach (var item in u.Items) { acceptedSubscriptionList.Add(item.Result); } }); GC.KeepAlive(firstSubscriptionTask); Thing thing; // wait until we know that connection was established for (var i = 0; i < 5; i++) { Assert.True(acceptedSubscriptionList.TryTake(out thing, 1000)); } Assert.False(acceptedSubscriptionList.TryTake(out thing, 50)); // open second subscription using (var rejectedSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId) { Strategy = SubscriptionOpeningStrategy.OpenIfFree, TimeToWaitBeforeConnectionRetry = TimeSpan.FromMilliseconds(2000) })) { // sometime not throwing (on linux) when written like this: // await Assert.ThrowsAsync<SubscriptionInUseException>(async () => await rejectedSubscription.StartAsync()); // so we put this in a try block try { await rejectedSubscription.Run(_ => { }); Assert.False(true, "Exepcted a throw here"); } catch (SubscriptionInUseException) { } } } } }
public async Task SubscriptionSimpleTakeOverStrategy() { using (var store = GetDocumentStore()) { await CreateDocuments(store, 1); var lastChangeVector = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector ?? null; await CreateDocuments(store, 5); var subscriptionCreationParams = new SubscriptionCreationOptions() { Query = "from Things", ChangeVector = lastChangeVector }; var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams); using ( var acceptedSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId))) { var acceptedSusbscriptionList = new BlockingCollection <Thing>(); var takingOverSubscriptionList = new BlockingCollection <Thing>(); long counter = 0; var batchProccessedByFirstSubscription = new AsyncManualResetEvent(); acceptedSubscription.AfterAcknowledgment += b => { if (Interlocked.Read(ref counter) == 5) { batchProccessedByFirstSubscription.Set(); } return(Task.CompletedTask); }; var firstRun = acceptedSubscription.Run(x => { foreach (var item in x.Items) { Interlocked.Increment(ref counter); acceptedSusbscriptionList.Add(item.Result); } }); Thing thing; // wait until we know that connection was established for (var i = 0; i < 5; i++) { Assert.True(acceptedSusbscriptionList.TryTake(out thing, 5000), "no doc"); } Assert.True(await batchProccessedByFirstSubscription.WaitAsync(TimeSpan.FromSeconds(15)), "no ack"); Assert.False(acceptedSusbscriptionList.TryTake(out thing)); // open second subscription using (var takingOverSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>( new SubscriptionWorkerOptions(subsId) { Strategy = SubscriptionOpeningStrategy.TakeOver })) { GC.KeepAlive(takingOverSubscription.Run(x => { foreach (var item in x.Items) { takingOverSubscriptionList.Add(item.Result); } })); Assert.ThrowsAsync <SubscriptionInUseException>(() => firstRun).Wait(); await CreateDocuments(store, 5); // wait until we know that connection was established for (var i = 0; i < 5; i++) { Assert.True(takingOverSubscriptionList.TryTake(out thing, 5000), "no doc takeover"); } Assert.False(takingOverSubscriptionList.TryTake(out thing)); } } } }
public async Task SubscriptionWaitStrategy() { using (var store = GetDocumentStore()) { await CreateDocuments(store, 1); var lastChangeVector = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector; var subscriptionCreationParams = new SubscriptionCreationOptions() { Query = "from Things", ChangeVector = lastChangeVector }; await CreateDocuments(store, 5); var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams); using ( var acceptedSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId))) { var acceptedSusbscriptionList = new BlockingCollection <Thing>(); var waitingSubscriptionList = new BlockingCollection <Thing>(); var ackSentAmre = new AsyncManualResetEvent(); acceptedSubscription.AfterAcknowledgment += b => { ackSentAmre.Set(); return(Task.CompletedTask); }; GC.KeepAlive(acceptedSubscription.Run(x => { foreach (var item in x.Items) { acceptedSusbscriptionList.Add(item.Result); } Thread.Sleep(20); })); // wait until we know that connection was established Thing thing; // wait until we know that connection was established for (var i = 0; i < 5; i++) { Assert.True(acceptedSusbscriptionList.TryTake(out thing, 50000)); } Assert.False(acceptedSusbscriptionList.TryTake(out thing, 50)); // open second subscription using ( var waitingSubscription = store.Subscriptions.GetSubscriptionWorker <Thing>(new SubscriptionWorkerOptions(subsId) { Strategy = SubscriptionOpeningStrategy.WaitForFree, TimeToWaitBeforeConnectionRetry = TimeSpan.FromMilliseconds(250) })) { GC.KeepAlive(waitingSubscription.Run(x => { foreach (var item in x.Items) { waitingSubscriptionList.Add(item.Result); } })); Assert.True(await ackSentAmre.WaitAsync(TimeSpan.FromSeconds(50))); acceptedSubscription.Dispose(); await CreateDocuments(store, 5); // wait until we know that connection was established for (var i = 0; i < 5; i++) { Assert.True(waitingSubscriptionList.TryTake(out thing, 3000)); } Assert.False(waitingSubscriptionList.TryTake(out thing, 50)); } } } }
public async Task CriteriaScriptWithTransformation(bool useSsl) { string dbName = GetDatabaseName(); X509Certificate2 clientCertificate = null; X509Certificate2 adminCertificate = null; if (useSsl) { var serverCertPath = SetupServerAuthentication(); adminCertificate = AskServerForClientCertificate(serverCertPath, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin); clientCertificate = AskServerForClientCertificate(serverCertPath, new Dictionary <string, DatabaseAccess> { [dbName] = DatabaseAccess.ReadWrite, }); } using (var store = GetDocumentStore(new Options { AdminCertificate = adminCertificate, ClientCertificate = clientCertificate, ModifyDatabaseName = s => dbName })) { using (var subscriptionManager = new DocumentSubscriptions(store)) { await CreateDocuments(store, 1); var lastChangeVector = (await store.Admin.SendAsync(new GetStatisticsOperation())).DatabaseChangeVector; await CreateDocuments(store, 6); var subscriptionCreationParams = new SubscriptionCreationOptions() { Query = @" declare function project(d) { var namSuffix = parseInt(d.Name.replace('ThingNo', '')); if (namSuffix <= 2){ return false; } else if (namSuffix == 3){ return null; } else if (namSuffix == 4){ return d; } return {Name: 'foo', OtherDoc:load('things/6-A')} } from Things as d select project(d) ", ChangeVector = lastChangeVector }; var subsId = subscriptionManager.Create(subscriptionCreationParams); using (var subscription = subscriptionManager.Open <BlittableJsonReaderObject>(new SubscriptionConnectionOptions(subsId))) { using (store.GetRequestExecutor().ContextPool.AllocateOperationContext(out JsonOperationContext context)) { var list = new BlockingCollection <BlittableJsonReaderObject>(); GC.KeepAlive(subscription.Run(x => { foreach (var item in x.Items) { list.Add(context.ReadObject(item.Result, "test")); } })); BlittableJsonReaderObject thing; Assert.True(list.TryTake(out thing, 5000)); dynamic dynamicThing = new DynamicBlittableJson(thing); Assert.Equal("ThingNo4", dynamicThing.Name); Assert.True(list.TryTake(out thing, 5000)); dynamicThing = new DynamicBlittableJson(thing); Assert.Equal("foo", dynamicThing.Name); Assert.Equal("ThingNo4", dynamicThing.OtherDoc.Name); Assert.False(list.TryTake(out thing, 50)); } } } } }
private async Task <RunResult> SingleTestRun(int workers, int fakeProcessingTimePerBatch, bool script, bool revision = false) { try { if (script == false && revision) { throw new InvalidOperationException("Can't have a revision without script"); } string revisions = revision ? " (Revisions = true)" : ""; if (script) { SubscriptionCreationOptions subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from " + _collectionName + revisions }; _subscriptionName = await _store.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false); Console.WriteLine($"Created Subscription with query: '{subscriptionCreationParams.Query}'"); } else { _subscriptionName = await _store.Subscriptions.CreateAsync <Order>(); Console.WriteLine($"Created Subscription without query"); } EnableRevisions(); var workersList = new List <SubscriptionWorker <dynamic> >(); var taskList = new List <Task>(); Stopwatch sp = null; for (int i = 0; i < workers; i++) { var worker = _store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(_subscriptionName) { Strategy = SubscriptionOpeningStrategy.Concurrent, TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5), MaxDocsPerBatch = _batchSize }); workersList.Add(worker); worker.OnEstablishedSubscriptionConnection += () => { Interlocked.CompareExchange(ref sp, Stopwatch.StartNew(), null); }; } var tcs = new TaskCompletionSource <RunResult>(); var sentCount = 0; foreach (var worker in workersList) { taskList.Add(worker.Run((async o => { if (Interlocked.Add(ref sentCount, o.Items.Count) >= _docsAmountToTest) { tcs.TrySetResult(new RunResult { DocsProccessed = sentCount, DocsRequested = _batchSize, ElapsedMs = sp.ElapsedMilliseconds }); } await Task.Delay(fakeProcessingTimePerBatch); }))); } var result = await tcs.Task.ConfigureAwait(false); foreach (var worker in workersList) { await worker.DisposeAsync().ConfigureAwait(false); } await Task.WhenAll(taskList); return(result); } catch (Exception ex) { Console.WriteLine(ex); throw; } }
public CreateSubscriptionCommand(SubscriptionCreationOptions options, string id = null) { _options = options; _id = id; }
public async Task CanDisableTcpCompressionOnTheClientViaStoreConventions() { var server = GetNewServer(new ServerCreationOptions { RunInMemory = false }); using (var store1 = GetDocumentStore(new Options { Server = server })) using (var store2 = GetDocumentStore(new Options { Server = server, ModifyDocumentStore = s => s.Conventions.DisableTcpCompression = true })) { Assert.False(server.Configuration.Server.DisableTcpCompression); foreach (var store in new [] { store1, store2 }) { var compressionDisabled = store == store2; using (var session = store.OpenSession()) { for (int i = 0; i < 10; i++) { session.Store(new User()); } session.SaveChanges(); } var subscriptionCreationParams = new SubscriptionCreationOptions() { Query = "from Users" }; var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams); using (var subscription = store.Subscriptions.GetSubscriptionWorker <User>(new SubscriptionWorkerOptions(subsId) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { var list = new BlockingCollection <User>(); GC.KeepAlive(subscription.Run(u => { foreach (var item in u.Items) { list.Add(item.Result); } })); User user; for (var i = 0; i < 10; i++) { Assert.True(list.TryTake(out user, 1000)); } Assert.False(list.TryTake(out user, 50)); var db = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database); var tcpConnections = db.RunningTcpConnections.ToList(); Assert.Equal(1, tcpConnections.Count); if (compressionDisabled) { Assert.False(tcpConnections[0].Stream is Sparrow.Utils.ReadWriteCompressedStream); } else { Assert.True(tcpConnections[0].Stream is Sparrow.Utils.ReadWriteCompressedStream); } } } } }
public CreateSubscriptionCommand(SubscriptionCreationOptions options) { _options = options; }
public async Task SubscribtionWithEtag() { using (var store = GetDocumentStore()) { var us1 = new User { Id = "users/1", Name = "john", Age = 22 }; var us2 = new User { Id = "users/2", Name = "KY", Age = 30 }; var us3 = new User { Id = "users/3", Name = "BEN", Age = 30 }; var us4 = new User { Id = "users/4", Name = "Hila", Age = 29 }; var us5 = new User { Id = "users/5", Name = "Revital", Age = 34 }; using (var session = store.OpenAsyncSession()) { await session.StoreAsync(us1); await session.StoreAsync(us2); await session.StoreAsync(us3); await session.StoreAsync(us4); await session.StoreAsync(us5); await session.SaveChangesAsync(); var user2ChangeVector = session.Advanced.GetChangeVectorFor(us2); var subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from Users", ChangeVector = user2ChangeVector }; var id = await store.Subscriptions.CreateAsync(subscriptionCreationParams); var users = new List <dynamic>(); using (var subscription = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(id) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { var docs = new BlockingCollection <dynamic>(); var keys = new BlockingCollection <string>(); var ages = new BlockingCollection <int>(); GC.KeepAlive(subscription.Run(u => { foreach (var x in u.Items) { keys.Add(x.Result.Id); ages.Add(x.Result.Age); docs.Add((x.Result)); } })); dynamic doc; Assert.True(docs.TryTake(out doc, _waitForDocTimeout)); users.Add(doc); Assert.True(docs.TryTake(out doc, _waitForDocTimeout)); users.Add(doc); Assert.True(docs.TryTake(out doc, _waitForDocTimeout)); users.Add(doc); var cnt = users.Count; Assert.Equal(3, cnt); string key; Assert.True(keys.TryTake(out key, _waitForDocTimeout)); Assert.Equal("users/3", key); Assert.True(keys.TryTake(out key, _waitForDocTimeout)); Assert.Equal("users/4", key); Assert.True(keys.TryTake(out key, _waitForDocTimeout)); Assert.Equal("users/5", key); int age; Assert.True(ages.TryTake(out age, _waitForDocTimeout)); Assert.Equal(30, age); Assert.True(ages.TryTake(out age, _waitForDocTimeout)); Assert.Equal(29, age); Assert.True(ages.TryTake(out age, _waitForDocTimeout)); Assert.Equal(34, age); } } } }
public async Task CanDisableTcpCompressionViaConfiguration_SubscriptionsTest() { var server = GetNewServer(new ServerCreationOptions { RunInMemory = false, CustomSettings = new Dictionary <string, string>() { [RavenConfiguration.GetKey(x => x.Server.DisableTcpCompression)] = "true" } }); Assert.True(server.Configuration.Server.DisableTcpCompression); using (var store = GetDocumentStore(new Options { Server = server })) { using (var session = store.OpenSession()) { for (int i = 0; i < 10; i++) { session.Store(new User()); } session.SaveChanges(); } var subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from Users" }; var subsId = await store.Subscriptions.CreateAsync(subscriptionCreationParams); using (var subscription = store.Subscriptions.GetSubscriptionWorker <User>(new SubscriptionWorkerOptions(subsId) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { var list = new BlockingCollection <User>(); GC.KeepAlive(subscription.Run(u => { foreach (var item in u.Items) { list.Add(item.Result); } })); User user; for (var i = 0; i < 10; i++) { Assert.True(list.TryTake(out user, 1000)); } Assert.False(list.TryTake(out user, 50)); var db = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database); var tcpConnections = db.RunningTcpConnections.ToList(); Assert.Equal(1, tcpConnections.Count); // assert non-compressed tcp connection Assert.False(tcpConnections[0].Stream is Sparrow.Utils.ReadWriteCompressedStream); } } }
public override void RunActualTest() { using (DocumentStore.Initialize()) { ReportInfo("Inserting products docs"); InsertProducts(); ReportInfo("Inserting shippers docs"); InsertShippers(); ReportInfo("Bulk insert users docs"); var bulkInsertUsersTask = Task.Run(() => BulkInsertUsersDocuments()); try { var usersSubscription = new SubscriptionCreationOptions <User2> { Name = $"UsersSubscription.{GenralGuid}", Filter = x => (x.Age % 2) == 0 }; var orderSubscription = new SubscriptionCreationOptions <User2> { Name = $"CreateOrderSubscription.{GenralGuid}", Filter = x => (x.Products != null), Projection = x => new { ProductsNames = x.Products } }; ReportInfo("Create subscriptions : UsersSubscription"); var createUsersSubscription = DocumentStore.Subscriptions.Create(usersSubscription); ReportInfo("Create subscriptions : CreateOrderSubscription"); var createOrderSubscription = DocumentStore.Subscriptions.Create(orderSubscription); var usersSubscriptionWorkerOptions = new SubscriptionWorkerOptions(createUsersSubscription) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5), MaxDocsPerBatch = 20, CloseWhenNoDocsLeft = false }; var orderSubscriptionWorkerOptions = new SubscriptionWorkerOptions(createOrderSubscription) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5), MaxDocsPerBatch = 3, CloseWhenNoDocsLeft = false }; ReportInfo("UsersSubscription: get subscriptions worker"); var usersSubscriptionWorker = DocumentStore.Subscriptions.GetSubscriptionWorker <dynamic>(usersSubscriptionWorkerOptions); ReportInfo("CreateOrderSubscription: get subscriptions worker"); var orderSubscriptionWorker = DocumentStore.Subscriptions.GetSubscriptionWorker <dynamic>(orderSubscriptionWorkerOptions); ReportInfo("Start inserting products to users"); var usersSubscriptionRun = Task.Run(() => InsertProductsToUsers(usersSubscriptionWorker)); ReportInfo("Start creating orders"); var orderSubscriptionRun = Task.Run(() => CreateOrderDoc(orderSubscriptionWorker)); var i = 0; foreach (var shipper in _shipperGuid) { var s = "shipper." + GenralGuid + "/" + shipper; var shipperSubscription = new SubscriptionCreationOptions <Order> { Name = $"shipperSubscription-{i}.{GenralGuid}", Filter = x => (x.ShipVia.StartsWith(s)) }; ReportInfo($"Create subscriptions : shipperSubscription-{i}.{GenralGuid}"); var createShipperSubscription = DocumentStore.Subscriptions.Create(shipperSubscription); var shipperSubscriptionWorkerOptions = new SubscriptionWorkerOptions(createShipperSubscription) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5), MaxDocsPerBatch = 6, CloseWhenNoDocsLeft = false }; ReportInfo($"shipperSubscription-{i}.{GenralGuid}: get subscriptions worker"); var shipperSubscriptionWorker = DocumentStore.Subscriptions.GetSubscriptionWorker <Order>(shipperSubscriptionWorkerOptions); var i1 = i; ReportInfo($"Start counting orders for shipper {i}. Id: {_shipperGuid[i]}"); var getShipperTask = Task.Run(() => GetShipper(shipperSubscriptionWorker, i1)); _tasks.AddLast(getShipperTask); i += 1; } } catch (Exception e) { ReportFailure("Error:", e); } while (_tasks.All(x => (x.IsCompleted)) == false) { } var success = true; for (int i = 0; i < ShippersCount; i++) { if (_shipper[i] != _shipperRes[i]) { ReportInfo($"{i}: {_shipper[i]} != {_shipperRes[i]}"); success = false; break; } } if (success) { ReportSuccess("Test done"); } else { ReportFailure("Test Failed", null); } } }
public async Task SubscribtionWithEtag_MultipleOpens() { using (var store = GetDocumentStore()) { var us1 = new User { Id = "users/1", Name = "john", Age = 22 }; var us2 = new User { Id = "users/2", Name = "KY", Age = 30 }; var us3 = new User { Id = "users/3", Name = "BEN", Age = 30 }; var us4 = new User { Id = "users/4", Name = "Hila", Age = 29 }; var us5 = new User { Id = "users/5", Name = "Revital", Age = 34 }; string subscriptionName; var subscriptionReleasedAwaiter = Task.CompletedTask; using (var session = store.OpenAsyncSession()) { await session.StoreAsync(us1); await session.StoreAsync(us2); await session.StoreAsync(us3); await session.StoreAsync(us4); await session.StoreAsync(us5); await session.SaveChangesAsync(); var user2ChangeVector = session.Advanced.GetChangeVectorFor(us2); var subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from Users", ChangeVector = user2ChangeVector }; subscriptionName = await store.Subscriptions.CreateAsync(subscriptionCreationParams); var users = new List <dynamic>(); using (var subscription = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(subscriptionName) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { var docs = new BlockingCollection <dynamic>(); var keys = new BlockingCollection <string>(); var ages = new BlockingCollection <int>(); var mre = new ManualResetEvent(false); subscription.AfterAcknowledgment += batch => { mre.Set(); return(Task.CompletedTask); }; GC.KeepAlive(subscription.Run(u => { foreach (var x in u.Items) { keys.Add(x.Result.Id); ages.Add(x.Result.Age); docs.Add((x.Result)); } })); var db = await GetDatabase(store.Database); var subscriptionState = db.SubscriptionStorage.GetSubscriptionFromServerStore(subscriptionName); subscriptionReleasedAwaiter = db.SubscriptionStorage.GetSubscriptionConnectionInUseAwaiter(subscriptionState.SubscriptionId); dynamic doc; Assert.True(docs.TryTake(out doc, _waitForDocTimeout)); users.Add(doc); Assert.True(docs.TryTake(out doc, _waitForDocTimeout)); users.Add(doc); Assert.True(docs.TryTake(out doc, _waitForDocTimeout)); users.Add(doc); var cnt = users.Count; Assert.Equal(3, cnt); string key; Assert.True(keys.TryTake(out key, _waitForDocTimeout)); Assert.Equal("users/3", key); Assert.True(keys.TryTake(out key, _waitForDocTimeout)); Assert.Equal("users/4", key); Assert.True(keys.TryTake(out key, _waitForDocTimeout)); Assert.Equal("users/5", key); int age; Assert.True(ages.TryTake(out age, _waitForDocTimeout)); Assert.Equal(30, age); Assert.True(ages.TryTake(out age, _waitForDocTimeout)); Assert.Equal(29, age); Assert.True(ages.TryTake(out age, _waitForDocTimeout)); Assert.Equal(34, age); Assert.True(mre.WaitOne(250)); } } Assert.True(Task.WaitAll(new[] { subscriptionReleasedAwaiter }, 250)); using (var subscription = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(subscriptionName) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { var docs = new BlockingCollection <dynamic>(); GC.KeepAlive(subscription.Run(x => { docs.Add((x)); })); dynamic item; var tryTake = docs.TryTake(out item, TimeSpan.FromMilliseconds(250)); Assert.False(tryTake); } } }
public async Task SubscriptionShouldRespectDocumentsWithCompressedData() { using (var documentStore = this.GetDocumentStore()) { Server.ServerStore.Observer.Suspended = true; var originalDoc = new Doc { Id = "doc/1", StrVal = new string(Enumerable.Repeat('.', 129).ToArray()), LongByteArray = Enumerable.Repeat((byte)2, 1024).ToArray() }; using (var session = documentStore.OpenAsyncSession()) { await session.StoreAsync(originalDoc); await session.SaveChangesAsync(); } var database = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(documentStore.Database); using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { var doc = database.DocumentsStorage.Get(context, "doc/1"); MemoryStream ms = new MemoryStream(); using (var newContext = JsonOperationContext.ShortTermSingleUse()) using (var writer = new BlittableJsonTextWriter(newContext, ms)) { writer.WriteDocument(newContext, doc, metadataOnly: false); writer.Flush(); var bjro = GetReaderFromMemoryStream(ms, context); var desereializedDoc = (Doc)EntityToBlittable.ConvertToEntity(typeof(Doc), null, bjro, DocumentConventions.Default); Assert.Equal(originalDoc.StrVal, desereializedDoc.StrVal); Assert.Equal(originalDoc.LongByteArray, originalDoc.LongByteArray); } } var subscriptionCreationParams = new SubscriptionCreationOptions { Query = "from Docs", }; var subsId = await documentStore.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false); var amre = new AsyncManualResetEvent(); using (var subscription = documentStore.Subscriptions.GetSubscriptionWorker <Doc>(new SubscriptionWorkerOptions(subsId))) { var t = subscription.Run(batch => { var receivedDoc = batch.Items.First().Result; Assert.Equal(originalDoc.LongByteArray, receivedDoc.LongByteArray); Assert.Equal(originalDoc.StrVal, receivedDoc.StrVal); amre.Set(); }); try { Assert.True(await amre.WaitAsync(TimeSpan.FromSeconds(60))); } catch { if (t.IsFaulted) { t.Wait(); } throw; } } } }
public CreateSubscriptionCommand(DocumentConventions conventions, SubscriptionCreationOptions options, string id = null) { _conventions = conventions; _options = options; _id = id; }
public CreateSubscriptionCommand(DocumentConventions conventions, SubscriptionCreationOptions options, string id = null) { _options = options ?? throw new ArgumentNullException(nameof(options)); _id = id; }