public void TestBatcherBatchSize5() { doneSignal = new CountdownEvent(10); inboxCapacity = 10; processorDelay = TimeSpan.FromSeconds(1); var scheduler = new SingleTaskThreadpoolScheduler(); var batcher = new Batcher <string>(new TaskFactory(scheduler), inboxCapacity, processorDelay, TestBatcherBatchSize5Processor); var objectsToQueue = new List <string>(); for (var i = 0; i < inboxCapacity * 10; i++) { objectsToQueue.Add(i.ToString()); if (objectsToQueue.Count == 5) { batcher.QueueObjects(objectsToQueue); objectsToQueue.Clear(); } } var success = doneSignal.Wait(TimeSpan.FromSeconds(35)); Assert.IsTrue(success); }
public void TestChangeTrackerGiveUp() { var factory = new MockHttpClientFactory(); var httpHandler = (MockHttpRequestHandler)factory.HttpHandler; httpHandler.AddResponderThrowExceptionAllRequests(); var changeTrackerFinishedSignal = new CountdownEvent(1); var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, null); client.HttpClientFactory = factory; var testUrl = GetReplicationURL(); var scheduler = new SingleTaskThreadpoolScheduler(); var remoteSession = new RemoteSession(new RemoteSessionContructorOptions { BaseUrl = testUrl, WorkExecutor = new TaskFactory(scheduler) }); remoteSession.SetupHttpClientFactory(client.HttpClientFactory, database, "giveup"); remoteSession.Setup(new ReplicationOptions()); var changeTracker = ChangeTrackerFactory.Create(new ChangeTrackerOptions { DatabaseUri = testUrl, Mode = ChangeTrackerMode.OneShot, IncludeConflicts = true, Client = client, RetryStrategy = new ExponentialBackoffStrategy(2), WorkExecutor = new TaskFactory(scheduler), RemoteSession = remoteSession }); changeTracker.Start(); Assert.IsTrue(changeTrackerFinishedSignal.Wait(TimeSpan.FromSeconds(30))); }
public void TestBatcherBatchSize5() { doneSignal = new CountdownEvent(10); inboxCapacity = 10; processorDelay = TimeSpan.FromSeconds(1); var scheduler = new SingleTaskThreadpoolScheduler(); var batcher = new Batcher <string>(new BatcherOptions <string> { WorkExecutor = new TaskFactory(scheduler), Capacity = inboxCapacity, Delay = processorDelay, Processor = TestBatcherBatchSize5Processor }); var objectsToQueue = new List <string>(); for (var i = 0; i < inboxCapacity * 10; i++) { objectsToQueue.Add(i.ToString()); if (objectsToQueue.Count == 5) { batcher.QueueObjects(objectsToQueue); objectsToQueue.Clear(); } } var success = doneSignal.Wait(TimeSpan.FromSeconds(35)); success.Should().BeTrue("beacuse otherwise the batcher didn't run the correct number of times"); }
private void TestChangeTrackerBackoff(MockHttpClientFactory httpClientFactory) { var changeTrackerFinishedSignal = new CountdownEvent(1); var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, null); client.HttpClientFactory = httpClientFactory; var testUrl = GetReplicationURL(); var scheduler = new SingleTaskThreadpoolScheduler(); var changeTracker = ChangeTrackerFactory.Create(new ChangeTrackerOptions { DatabaseUri = testUrl, Mode = ChangeTrackerMode.LongPoll, IncludeConflicts = true, Client = client, RetryStrategy = new ExponentialBackoffStrategy(2), WorkExecutor = new TaskFactory(scheduler) }); changeTracker.Continuous = true; changeTracker.Start(); // sleep for a few seconds Sleep(8 * 1000); // make sure we got less than 10 requests in those 8 seconds (if it was hammering, we'd get a lot more) var handler = client.HttpRequestHandler; Assert.Less(handler.CapturedRequests.Count, 10); Assert.Greater(changeTracker.Backoff.NumAttempts, 0, String.Format("Observed attempts: {0}", changeTracker.Backoff.NumAttempts)); handler.ClearResponders(); handler.AddResponderReturnEmptyChangesFeed(); // at this point, the change tracker backoff should cause it to sleep for about 3 seconds // and so lets wait 3 seconds until it wakes up and starts getting valid responses Sleep(10 * 1000); // now find the delta in requests received in a 2s period int before = handler.CapturedRequests.Count; Sleep(2 * 1000); int after = handler.CapturedRequests.Count; // assert that the delta is high, because at this point the change tracker should // be hammering away Assert.Greater((after - before), 25); // the backoff numAttempts should have been reset to 0 Assert.IsTrue(changeTracker.Backoff.NumAttempts == 0); changeTracker.Stop(); var success = changeTrackerFinishedSignal.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); }
private void RunChangeTrackerTransientError( ChangeTrackerMode mode, Int32 errorCode, string statusMessage, Int32 numExpectedChangeCallbacks) { var changeTrackerFinishedSignal = new CountdownEvent(1); var changeReceivedSignal = new CountdownEvent(numExpectedChangeCallbacks); var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, changeReceivedSignal); MockHttpRequestHandler.HttpResponseDelegate sentinal = RunChangeTrackerTransientErrorDefaultResponder(); var responders = new List <MockHttpRequestHandler.HttpResponseDelegate>(); responders.Add(RunChangeTrackerTransientErrorDefaultResponder()); responders.Add(MockHttpRequestHandler.TransientErrorResponder(errorCode, statusMessage)); MockHttpRequestHandler.HttpResponseDelegate chainResponder = (request) => { if (responders.Count > 0) { var responder = responders[0]; responders.RemoveAt(0); return(responder(request)); } return(sentinal(request)); }; var handler = client.HttpRequestHandler; handler.SetResponder("_changes", chainResponder); var testUrl = GetReplicationURL(); var scheduler = new SingleTaskThreadpoolScheduler(); var changeTracker = ChangeTrackerFactory.Create(new ChangeTrackerOptions { DatabaseUri = testUrl, Mode = mode, IncludeConflicts = false, Client = client, RetryStrategy = new ExponentialBackoffStrategy(2), WorkExecutor = new TaskFactory(scheduler) }); changeTracker.Start(); var success = changeReceivedSignal.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); changeTracker.Stop(); success = changeTrackerFinishedSignal.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); }
private void ChangeTrackerTestWithMode(ChangeTrackerMode mode) { var changeTrackerFinishedSignal = new CountdownEvent(1); var changeReceivedSignal = new CountdownEvent(1); var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, changeReceivedSignal); client.ReceivedChangeDelegate = (IDictionary <string, object> change) => { Assert.IsTrue(change.ContainsKey("seq")); Assert.AreEqual("1", change["seq"]); }; var handler = client.HttpRequestHandler; handler.SetResponder("_changes", (request) => { var json = "{\"results\":[\n" + "{\"seq\":\"1\",\"id\":\"doc1-138\",\"changes\":[{\"rev\":\"1-82d\"}]}],\n" + "\"last_seq\":\"*:50\"}"; return(MockHttpRequestHandler.GenerateHttpResponseMessage(System.Net.HttpStatusCode.OK, null, json)); }); var testUrl = GetReplicationURL(); var scheduler = new SingleTaskThreadpoolScheduler(); var remoteSession = new RemoteSession(new RemoteSessionContructorOptions { WorkExecutor = new TaskFactory(scheduler), BaseUrl = testUrl }); remoteSession.SetupHttpClientFactory(client.HttpClientFactory, database, "testwithmode"); remoteSession.Setup(new ReplicationOptions()); var changeTracker = ChangeTrackerFactory.Create(new ChangeTrackerOptions { DatabaseUri = testUrl, Mode = mode, IncludeConflicts = false, Client = client, RetryStrategy = new ExponentialBackoffStrategy(2), WorkExecutor = new TaskFactory(scheduler), RemoteSession = remoteSession }); changeTracker.ActiveOnly = true; changeTracker.Start(); var success = changeReceivedSignal.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); changeTracker.Stop(); success = changeTrackerFinishedSignal.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); }
public void TestBatcherCancel() { var mre = new ManualResetEventSlim(); var scheduler = new SingleTaskThreadpoolScheduler(); var batcher = new Batcher <int>(new TaskFactory(scheduler), 5, 500, (inbox) => { mre.Set(); }); batcher.QueueObject(0); batcher.Clear(); Assert.False(mre.Wait(TimeSpan.FromSeconds(1)), "Batcher ran after being cancelled"); }
private void RunChangeTrackerTransientError( ChangeTrackerMode mode, Int32 errorCode, string statusMessage, Int32 numExpectedChangeCallbacks) { var changeTrackerFinishedSignal = new CountDownLatch(1); var changeReceivedSignal = new CountDownLatch(numExpectedChangeCallbacks); var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, changeReceivedSignal); MockHttpRequestHandler.HttpResponseDelegate sentinal = RunChangeTrackerTransientErrorDefaultResponder(); var responders = new List <MockHttpRequestHandler.HttpResponseDelegate>(); responders.Add(RunChangeTrackerTransientErrorDefaultResponder()); responders.Add(MockHttpRequestHandler.TransientErrorResponder(errorCode, statusMessage)); MockHttpRequestHandler.HttpResponseDelegate chainResponder = (request) => { if (responders.Count > 0) { var responder = responders[0]; responders.RemoveAt(0); return(responder(request)); } return(sentinal(request)); }; var handler = client.HttpRequestHandler; handler.SetResponder("_changes", chainResponder); var testUrl = GetReplicationURL(); var scheduler = new SingleTaskThreadpoolScheduler(); var changeTracker = new ChangeTracker(testUrl, mode, 0, false, client, new TaskFactory(scheduler)); changeTracker.UsePost = IsSyncGateway(testUrl); changeTracker.Start(); var success = changeReceivedSignal.Await(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); changeTracker.Stop(); success = changeTrackerFinishedSignal.Await(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); }
private void TestChangeTrackerBackoff(MockHttpClientFactory httpClientFactory) { var changeTrackerFinishedSignal = new CountDownLatch(1); var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, null); client.HttpClientFactory = httpClientFactory; var testUrl = GetReplicationURL(); var scheduler = new SingleTaskThreadpoolScheduler(); var changeTracker = new ChangeTracker(testUrl, ChangeTrackerMode.LongPoll, 0, false, client, new TaskFactory(scheduler)); changeTracker.UsePost = IsSyncGateway(testUrl); changeTracker.Start(); // sleep for a few seconds Sleep(10 * 1000); // make sure we got less than 10 requests in those 10 seconds (if it was hammering, we'd get a lot more) var handler = client.HttpRequestHandler; Assert.IsTrue(handler.CapturedRequests.Count < 25); Assert.IsTrue(changeTracker.backoff.NumAttempts > 0, "Observed attempts: {0}".Fmt(changeTracker.backoff.NumAttempts)); handler.ClearResponders(); handler.AddResponderReturnEmptyChangesFeed(); // at this point, the change tracker backoff should cause it to sleep for about 3 seconds // and so lets wait 3 seconds until it wakes up and starts getting valid responses Sleep(3 * 1000); // now find the delta in requests received in a 2s period int before = handler.CapturedRequests.Count; Sleep(2 * 1000); int after = handler.CapturedRequests.Count; // assert that the delta is high, because at this point the change tracker should // be hammering away Assert.IsTrue((after - before) > 25, "{0} <= 25", (after - before)); // the backoff numAttempts should have been reset to 0 Assert.IsTrue(changeTracker.backoff.NumAttempts == 0); changeTracker.Stop(); var success = changeTrackerFinishedSignal.Await(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); }
public void TestBatcherAddAfterCancel() { var evt = new CountdownEvent(2); var scheduler = new SingleTaskThreadpoolScheduler(); var batcher = new Batcher <int>(new TaskFactory(scheduler), 5, 500, (inbox) => { evt.Signal(); }); batcher.QueueObject(0); batcher.Clear(); batcher.QueueObject(0); Assert.False(evt.Wait(TimeSpan.FromSeconds(1)), "Batcher ran too many times"); Assert.True(evt.CurrentCount == 1, "Batcher never ran"); }
public void TestBatcherCancel() { var mre = new ManualResetEventSlim(); var scheduler = new SingleTaskThreadpoolScheduler(); var batcher = new Batcher <int>(new BatcherOptions <int> { WorkExecutor = new TaskFactory(scheduler), Capacity = 5, Delay = TimeSpan.FromMilliseconds(500), Processor = (inbox) => mre.Set() }); batcher.QueueObject(0); mre.Wait(1000).Should().BeTrue("because otherwise the batcher didn't initially run"); mre.Reset(); batcher.QueueObject(0); batcher.Clear(); mre.Wait(TimeSpan.FromSeconds(1)).Should().BeFalse("because otherwise the batcher ran after being cancelled"); }
public void TestBatcherAddAfterCancel() { var evt = new CountdownEvent(1); var scheduler = new SingleTaskThreadpoolScheduler(); var batcher = new Batcher <int>(new BatcherOptions <int> { WorkExecutor = new TaskFactory(scheduler), Capacity = 5, Delay = TimeSpan.FromMilliseconds(500), Processor = (inbox) => evt.Signal() }); batcher.QueueObject(0); evt.Wait(1000).Should().BeTrue("because otherwise the batcher didn't initially run"); evt.Reset(2); batcher.QueueObject(0); batcher.Clear(); batcher.QueueObject(0); evt.Wait(TimeSpan.FromSeconds(1.5)).Should().BeFalse("because otherwise the batcher ran too many times"); evt.CurrentCount.Should().Be(1, "because otherwise the batcher never ran"); }
private void ChangeTrackerTestWithMode(ChangeTrackerMode mode) { var changeTrackerFinishedSignal = new CountDownLatch(1); var changeReceivedSignal = new CountDownLatch(1); var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, changeReceivedSignal); client.ReceivedChangeDelegate = (IDictionary <string, object> change) => { Assert.IsTrue(change.ContainsKey("seq")); Assert.AreEqual("1", change["seq"]); }; var handler = client.HttpRequestHandler; handler.SetResponder("_changes", (request) => { var json = "{\"results\":[\n" + "{\"seq\":\"1\",\"id\":\"doc1-138\",\"changes\":[{\"rev\":\"1-82d\"}]}],\n" + "\"last_seq\":\"*:50\"}"; return(MockHttpRequestHandler.GenerateHttpResponseMessage(System.Net.HttpStatusCode.OK, null, json)); }); var testUrl = GetReplicationURL(); var scheduler = new SingleTaskThreadpoolScheduler(); var changeTracker = new ChangeTracker(testUrl, mode, 0, false, client, new TaskFactory(scheduler)); changeTracker.UsePost = IsSyncGateway(testUrl); changeTracker.Start(); var success = changeReceivedSignal.Await(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); changeTracker.Stop(); success = changeTrackerFinishedSignal.Await(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); }
// This is used by the listener internal Replication ReplicationWithProperties(IDictionary <string, object> properties) { // Extract the parameters from the JSON request body: // http://wiki.apache.org/couchdb/Replication bool push, createTarget; var results = new Dictionary <string, object>() { { "database", null }, { "remote", null }, { "headers", null }, { "authorizer", null } }; Status result = ParseReplicationProperties(properties, out push, out createTarget, results); if (result.IsError) { throw Misc.CreateExceptionAndLog(Log.To.Listener, result.Code, TAG, "Failed to create replication"); } object continuousObj = properties.Get("continuous"); bool continuous = false; if (continuousObj is bool) { continuous = (bool)continuousObj; } var scheduler = new SingleTaskThreadpoolScheduler(); Replication rep = null; if (push) { rep = new Pusher((Database)results["database"], (Uri)results["remote"], continuous, new TaskFactory(scheduler)); } else { rep = new Puller((Database)results["database"], (Uri)results["remote"], continuous, new TaskFactory(scheduler)); } rep.Filter = properties.Get("filter") as string; rep.FilterParams = properties.Get("query_params").AsDictionary <string, object>(); rep.DocIds = properties.Get("doc_ids").AsList <string>(); rep.Headers = new Dictionary <string, string>(); foreach (var header in results.Get("headers").AsDictionary <string, string>()) { if (header.Key.ToLowerInvariant() == "cookie") { var cookie = default(Cookie); if (CookieParser.TryParse(header.Value, ((Uri)results["remote"]).GetLeftPart(UriPartial.Authority), out cookie)) { rep.SetCookie(cookie.Name, cookie.Value, cookie.Path, cookie.Expires, cookie.Secure, cookie.HttpOnly); } else { Log.To.Listener.W(TAG, "Invalid cookie string received ({0}), ignoring...", header.Value); } } else { rep.Headers.Add(header.Key, header.Value); } } rep.Headers = results.Get("headers").AsDictionary <string, string>(); rep.Authenticator = results.Get("authorizer") as IAuthenticator; if (push) { ((Pusher)rep).CreateTarget = createTarget; } var db = (Database)results["database"]; // If this is a duplicate, reuse an existing replicator: var activeReplicators = default(IList <Replication>); var existing = default(Replication); if (db.ActiveReplicators.AcquireTemp(out activeReplicators)) { existing = activeReplicators.FirstOrDefault(x => x.LocalDatabase == rep.LocalDatabase && x.RemoteUrl == rep.RemoteUrl && x.IsPull == rep.IsPull && x.RemoteCheckpointDocID().Equals(rep.RemoteCheckpointDocID())); } return(existing ?? rep); }
internal Replication ReplicationWithProperties(IDictionary <string, object> properties) { // Extract the parameters from the JSON request body: // http://wiki.apache.org/couchdb/Replication bool push, createTarget; var results = new Dictionary <string, object>() { { "database", null }, { "remote", null }, { "headers", null }, { "authorizer", null } }; Status result = ParseReplicationProperties(properties, out push, out createTarget, results); if (result.IsError) { throw new CouchbaseLiteException(result.Code); } object continuousObj = properties.Get("continuous"); bool continuous = false; if (continuousObj is bool) { continuous = (bool)continuousObj; } var scheduler = new SingleTaskThreadpoolScheduler(); Replication rep = null; if (push) { rep = new Pusher((Database)results["database"], (Uri)results["remote"], continuous, new TaskFactory(scheduler)); } else { rep = new Puller((Database)results["database"], (Uri)results["remote"], continuous, new TaskFactory(scheduler)); } rep.Filter = properties.Get("filter") as string; rep.FilterParams = properties.Get("query_params") as IDictionary <string, object>; rep.DocIds = properties.Get("doc_ids") as IEnumerable <string>; rep.RequestHeaders = results.Get("headers") as IDictionary <string, object>; rep.Authenticator = results.Get("authorizer") as IAuthenticator; if (push) { ((Pusher)rep).CreateTarget = createTarget; } var db = (Database)results["database"]; // If this is a duplicate, reuse an existing replicator: var existing = db.ActiveReplicators.FirstOrDefault(x => x.LocalDatabase == rep.LocalDatabase && x.RemoteUrl == rep.RemoteUrl && x.IsPull == rep.IsPull && x.RemoteCheckpointDocID().Equals(rep.RemoteCheckpointDocID())); return(existing ?? rep); }