public void TestFailedBulkGetDoesntChangeLastSequence() { if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"])) { Assert.Inconclusive("Replication tests disabled."); return; } var fakeFactory = new MockHttpClientFactory(false); fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) => { var str = request.Content.ReadAsStringAsync().Result; if(str.IndexOf("doc2") != -1) { Log.D(Tag, "Rejecting this bulk get because it looks like the first batch"); throw new OperationCanceledException(); } Log.D(Tag, "Letting this bulk get through"); return new RequestCorrectHttpMessage(); }); manager.DefaultHttpClientFactory = fakeFactory; Manager.DefaultOptions.MaxRevsToGetInBulk = 10; Manager.DefaultOptions.MaxOpenHttpConnections = 8; Manager.DefaultOptions.RequestTimeout = TimeSpan.FromSeconds(5); CreatePullAndTest((int)(Manager.DefaultOptions.MaxRevsToGetInBulk * 1.5), repl => { Log.D(Tag, "Document count increased to {0} with last sequence '{1}'", database.DocumentCount, repl.LastSequence); Assert.IsTrue(database.DocumentCount > 0, "Didn't get docs from second bulk get batch"); Assert.AreEqual("0", repl.LastSequence, "LastSequence was advanced"); }); fakeFactory.HttpHandler.ClearResponders(); var pull = database.CreatePullReplication(GetReplicationURL()); RunReplication(pull); Assert.AreEqual(pull.ChangesCount, pull.CompletedChangesCount); Assert.AreNotEqual(pull.LastSequence, "0"); }
public void TestHeaders() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } var mockHttpClientFactory = new MockHttpClientFactory(false); manager.DefaultHttpClientFactory = mockHttpClientFactory; var mockHttpHandler = mockHttpClientFactory.HttpHandler; using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var remote = remoteDb.RemoteUri; var puller = database.CreatePullReplication(remote); var headers = new Dictionary<string, string>(); headers["foo"] = "bar"; var pusher = database.CreatePushReplication (remote); pusher.Headers = headers; pusher.Start (); Sleep (5000); pusher.Stop (); ValidateHttpHeaders (mockHttpHandler); mockHttpHandler.ClearCapturedRequests (); puller.Headers = headers; puller.Start(); Sleep(5000); puller.Stop(); ValidateHttpHeaders (mockHttpHandler); } }
public void TestServerDoesNotSupportMultipart() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } Assert.AreEqual(0, database.GetLastSequenceNumber()); var properties1 = new Dictionary<string, object>() { { "dynamic", 1 } }; var doc = CreateDocumentWithProperties(database, properties1); var rev1 = doc.CurrentRevision; var unsavedRev2 = doc.CreateRevision(); var attachmentStream = GetAsset("attachment.png"); unsavedRev2.SetAttachment("attachment.png", "image/png", attachmentStream); var rev2 = unsavedRev2.Save(); attachmentStream.Dispose(); Assert.IsNotNull(rev2); unsavedRev2.Dispose(); var httpClientFactory = new MockHttpClientFactory(); var httpHandler = httpClientFactory.HttpHandler; httpHandler.AddResponderFakeLocalDocumentUpdate404(); var responders = new List<MockHttpRequestHandler.HttpResponseDelegate>(); responders.Add((request) => { var json = "{\"error\":\"Unsupported Media Type\",\"reason\":\"missing\"}"; return MockHttpRequestHandler.GenerateHttpResponseMessage(HttpStatusCode.UnsupportedMediaType, "Unsupported Media Type", json); }); responders.Add((request) => { var props = new Dictionary<string, object>() { {"id", doc.Id}, {"ok", true}, {"rev", doc.CurrentRevisionId} }; return MockHttpRequestHandler.GenerateHttpResponseMessage(props); }); MockHttpRequestHandler.HttpResponseDelegate chainResponder = (request) => { if (responders.Count > 0) { var responder = responders[0]; responders.RemoveAt(0); return responder(request); } return null; }; httpHandler.SetResponder(doc.Id, chainResponder); manager.DefaultHttpClientFactory = httpClientFactory; using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var pusher = database.CreatePushReplication(remoteDb.RemoteUri); RunReplication(pusher); var count = 0; foreach (var request in httpHandler.CapturedRequests) { if (request.Method == HttpMethod.Put && request.RequestUri.PathAndQuery.Contains(doc.Id)) { var isMultipartContent = (request.Content is MultipartContent); if (count == 0) { Assert.IsTrue(isMultipartContent); } else { Assert.IsFalse(isMultipartContent); } count++; } } } }
public void TestBulkPullPermanentExceptionSurrender() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } Log.Domains.Sync.Level = Log.LogLevel.Debug; var fakeFactory = new MockHttpClientFactory(false); FlowControl flow = new FlowControl(new FlowItem[] { new ExceptionThrower(new SocketException()) { ExecutionCount = -1 }, }); fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) => flow.ExecuteNext<HttpResponseMessage>()); manager.DefaultHttpClientFactory = fakeFactory; using (var remoteDb = _sg.CreateDatabase(TempDbName())) { CreatePullAndTest(20, remoteDb, repl => Assert.IsTrue(database.GetDocumentCount() < 20, "Somehow got all the docs")); } }
public void TestContinuousReplicationErrorNotification() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } var httpClientFactory = new MockHttpClientFactory(); manager.DefaultHttpClientFactory = httpClientFactory; var httpHandler = httpClientFactory.HttpHandler; httpHandler.AddResponderThrowExceptionAllRequests(); using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var pusher = database.CreatePushReplication(remoteDb.RemoteUri); pusher.Continuous = true; var signal = new CountdownEvent(1); var observer = new ReplicationErrorObserver(signal); pusher.Changed += observer.Changed; pusher.Start(); CreateDocuments(database, 10); var success = signal.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); StopReplication(pusher); } }
private void RunPushReplicationWithTransientError(Int32 errorCode, string statusMessage, Boolean expectError) { var properties1 = new Dictionary<string, object>() { {"doc1", "testPushReplicationTransientError"} }; CreateDocumentWithProperties(database, properties1); var httpClientFactory = new MockHttpClientFactory(false); var httpHandler = httpClientFactory.HttpHandler; manager.DefaultHttpClientFactory = httpClientFactory; MockHttpRequestHandler.HttpResponseDelegate sentinal = MockHttpRequestHandler.FakeBulkDocs; var responders = new List<MockHttpRequestHandler.HttpResponseDelegate>(); responders.Add(MockHttpRequestHandler.TransientErrorResponder(errorCode, statusMessage)); MockHttpRequestHandler.HttpResponseDelegate chainResponder = (request) => { if (responders.Count > 0) { var responder = responders[0]; responders.RemoveAt(0); return responder(request); } return sentinal(request); }; httpHandler.SetResponder("_bulk_docs", chainResponder); // Create a replication observer to wait until replication finishes var replicationDoneSignal = new CountdownEvent(1); var replicationFinishedObserver = new ReplicationObserver(replicationDoneSignal); using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var pusher = database.CreatePushReplication(remoteDb.RemoteUri); pusher.Changed += replicationFinishedObserver.Changed; // save the checkpoint id for later usage var checkpointId = pusher.RemoteCheckpointDocID(); // kick off the replication pusher.Start(); // wait for it to finish var success = replicationDoneSignal.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); if (expectError) { Assert.IsNotNull(pusher.LastError); } else { Assert.IsNull(pusher.LastError); } // workaround for the fact that the replicationDoneSignal.Await() call could unblock before all // the statements in Replication.Stopped() have even had a chance to execute. Sleep(500); var localLastSequence = database.LastSequenceWithCheckpointId(checkpointId); if (expectError) { Assert.Null(localLastSequence); } else { Assert.IsNotNull(localLastSequence); } } }
public void TestAllLeafRevisionsArePushed() { var httpClientFactory = new MockHttpClientFactory(); var httpHandler = httpClientFactory.HttpHandler; httpHandler.AddResponderRevDiffsAllMissing(); httpHandler.AddResponderFakeLocalDocumentUpdate404(); httpHandler.ResponseDelayMilliseconds = 250; manager.DefaultHttpClientFactory = httpClientFactory; var doc = database.CreateDocument(); var rev1a = doc.CreateRevision().Save(); var rev2a = rev1a.CreateRevision().Save(); var rev3a = rev2a.CreateRevision().Save(); // delete the branch we've been using, then create a new one to replace it var rev4a = rev3a.DeleteDocument(); var rev2b = rev1a.CreateRevision().Save(true); Assert.AreEqual(rev2b.Id, doc.CurrentRevisionId); // sync with remote DB -- should push both leaf revisions var pusher = database.CreatePushReplication(GetReplicationURL()); RunReplication(pusher); Assert.IsNull(pusher.LastError); var foundRevsDiff = false; var capturedRequests = httpHandler.CapturedRequests; foreach (var httpRequest in capturedRequests) { var uriString = httpRequest.RequestUri.ToString(); if (uriString.EndsWith("_revs_diff", StringComparison.Ordinal)) { foundRevsDiff = true; var jsonMap = MockHttpRequestHandler.GetJsonMapFromRequest(httpRequest); var revisionIds = ((JArray)jsonMap.Get(doc.Id)).Values<string>().ToList(); Assert.AreEqual(2, revisionIds.Count); Assert.IsTrue(revisionIds.Contains(rev4a.Id)); Assert.IsTrue(revisionIds.Contains(rev2b.Id)); } } Assert.IsTrue(foundRevsDiff); }
[Test] // This test takes nearly 5 minutes to run, so only run when needed #endif public void TestLongRemovedChangesFeed() { var random = new Random(); var changesFeed = new StringBuilder("{\"results\":["); const int limit = 100000; HashSet<string> removedIDSet = new HashSet<string>(); for (var i = 1; i < limit; i++) { var removed = random.NextDouble() >= 0.5; if (removed) { var removedID = Misc.CreateGUID(); changesFeed.AppendFormat("{{\"seq\":\"{0}\",\"id\":\"{1}\",\"removed\":[\"fake\"]," + "\"changes\":[{{\"rev\":\"1-deadbeef\"}}]}},", i, removedID); removedIDSet.Add(removedID); } else { changesFeed.AppendFormat("{{\"seq\":\"{0}\",\"id\":\"{1}\",\"changes\":[{{\"rev\":\"1-deadbeef\"}}]}},", i, Misc.CreateGUID()); } } changesFeed.AppendFormat("{{\"seq\":\"{0}\",\"id\":\"{1}\",\"changes\":[{{\"rev\":\"1-deadbeef\"}}]}}]," + "last_seq: \"{0}\"}}", limit, Misc.CreateGUID()); var factory = new MockHttpClientFactory(false); factory.HttpHandler.SetResponder("_changes", req => { var response = new HttpResponseMessage(HttpStatusCode.OK); var changesString = changesFeed.ToString(); response.Content = new StringContent(changesString); return response; }); factory.HttpHandler.SetResponder("_bulk_get", req => { var contentStream = req.Content.ReadAsStreamAsync().Result; var content = Manager.GetObjectMapper().ReadValue<IDictionary<string, object>>(contentStream); var responseBody = new StringBuilder("--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\r\n"); foreach(var obj in content["docs"] as IEnumerable) { var dict = obj.AsDictionary<string, object>(); var nonexistent = removedIDSet.Contains(dict.GetCast<string>("id")); if(nonexistent) { return new HttpResponseMessage(HttpStatusCode.InternalServerError); // Just so we can know } else { responseBody.Append("Content-Type: application/json\r\n\r\n"); responseBody.AppendFormat("{{\"_id\":\"{0}\",\"_rev\":\"1-deadbeef\",\"foo\":\"bar\"}}\r\n", dict["id"]); } responseBody.Append("--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\r\n"); } responseBody.Remove(responseBody.Length - 2, 2); responseBody.Append("--"); var retVal = new HttpResponseMessage(HttpStatusCode.OK); var responseString = responseBody.ToString(); retVal.Content = new StringContent(responseString); retVal.Content.Headers.Remove("Content-Type"); retVal.Content.Headers.TryAddWithoutValidation("Content-Type", "multipart/mixed; boundary=\"67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\""); return retVal; }); manager.DefaultHttpClientFactory = factory; using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var puller = database.CreatePullReplication(remoteDb.RemoteUri); RunReplication(puller); Assert.AreEqual(ReplicationStatus.Stopped, puller.Status); Assert.AreNotEqual(limit, puller.ChangesCount); Assert.AreNotEqual(limit, puller.CompletedChangesCount); Assert.AreEqual(limit.ToString(), puller.LastSequence); } Sleep(1000); }
public ChangeTrackerTestClient(CountDownLatch stoppedSignal, CountDownLatch changedSignal) { this.stoppedSignal = stoppedSignal; this.changedSignal = changedSignal; HttpClientFactory = new MockHttpClientFactory(); }
public void TestHeaders() { var mockHttpClientFactory = new MockHttpClientFactory(); manager.DefaultHttpClientFactory = mockHttpClientFactory; var mockHttpHandler = mockHttpClientFactory.HttpHandler; mockHttpHandler.AddResponderThrowExceptionAllRequests(); Uri remote = GetReplicationURL(); Replication puller = database.CreatePullReplication(remote); var headers = new Dictionary<string, string>(); headers["foo"] = "bar"; puller.Headers = headers; RunReplication(puller); Assert.IsNotNull(puller.LastError); var foundFooHeader = false; var requests = mockHttpHandler.CapturedRequests; foreach (var request in requests) { var requestHeaders = request.Headers.GetValues("foo"); foreach (var requestHeader in requestHeaders) { foundFooHeader = true; Assert.AreEqual("bar", requestHeader); } } Assert.IsTrue(foundFooHeader); }
public void TestChangeTrackerBackoffInvalidJson() { var factory = new MockHttpClientFactory(); var httpHandler = (MockHttpRequestHandler)factory.HttpHandler; httpHandler.AddResponderReturnInvalidChangesFeedJson(); TestChangeTrackerBackoff(factory); }
public void TestChangeTrackerBackoffExceptions() { var factory = new MockHttpClientFactory(); var httpHandler = (MockHttpRequestHandler)factory.HttpHandler; httpHandler.AddResponderThrowExceptionAllRequests(); TestChangeTrackerBackoff(factory); }
private void TestChangeTrackerBackoff(MockHttpClientFactory httpClientFactory) { var changeTrackerFinishedSignal = new CountDownLatch(1); var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, null); client.HttpClientFactory = httpClientFactory; var testUrl = GetReplicationURL(); var scheduler = new SingleTaskThreadpoolScheduler(); var changeTracker = new ChangeTracker(testUrl, ChangeTrackerMode.LongPoll, 0, false, client, new TaskFactory(scheduler)); changeTracker.UsePost = IsSyncGateway(testUrl); changeTracker.Start(); // sleep for a few seconds Thread.Sleep(15 * 1000); // make sure we got less than 10 requests in those 10 seconds (if it was hammering, we'd get a lot more) var handler = client.HttpRequestHandler; Assert.IsTrue(handler.CapturedRequests.Count < 25); Assert.IsTrue(changeTracker.backoff.NumAttempts > 0, "Observed attempts: {0}".Fmt(changeTracker.backoff.NumAttempts)); handler.ClearResponders(); handler.AddResponderReturnEmptyChangesFeed(); // at this point, the change tracker backoff should cause it to sleep for about 3 seconds // and so lets wait 3 seconds until it wakes up and starts getting valid responses Thread.Sleep(3 * 1000); // now find the delta in requests received in a 2s period int before = handler.CapturedRequests.Count; Thread.Sleep(2 * 1000); int after = handler.CapturedRequests.Count; // assert that the delta is high, because at this point the change tracker should // be hammering away Assert.IsTrue((after - before) > 25); // the backoff numAttempts should have been reset to 0 Assert.IsTrue(changeTracker.backoff.NumAttempts == 0); changeTracker.Stop(); var success = changeTrackerFinishedSignal.Await(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); }
public void TestAllLeafRevisionsArePushed() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } var httpClientFactory = new MockHttpClientFactory(); var httpHandler = httpClientFactory.HttpHandler; httpHandler.AddResponderRevDiffsAllMissing(); httpHandler.AddResponderFakeLocalDocumentUpdate404(); httpHandler.ResponseDelayMilliseconds = 250; manager.DefaultHttpClientFactory = httpClientFactory; var doc = database.CreateDocument(); var rev1a = doc.CreateRevision().Save(); var rev2a = rev1a.CreateRevision().Save(); var rev3a = rev2a.CreateRevision().Save(); // delete the branch we've been using, then create a new one to replace it var rev4a = rev3a.DeleteDocument(); var unsaved = rev1a.CreateRevision(); unsaved.SetUserProperties(new Dictionary<string, object> { { "foo", "bar" } }); var rev2b = unsaved.Save(true); Assert.AreEqual(rev2b.Id, doc.CurrentRevisionId); using (var remoteDb = _sg.CreateDatabase(TempDbName())) { // sync with remote DB -- should push both leaf revisions var pusher = database.CreatePushReplication(remoteDb.RemoteUri); RunReplication(pusher); Assert.IsNull(pusher.LastError); var foundRevsDiff = false; var capturedRequests = httpHandler.CapturedRequests; foreach (var httpRequest in capturedRequests) { var uriString = httpRequest.RequestUri.ToString(); if (uriString.EndsWith("_revs_diff", StringComparison.Ordinal)) { foundRevsDiff = true; var jsonMap = MockHttpRequestHandler.GetJsonMapFromRequest(httpRequest); var revisionIds = ((JArray)jsonMap.Get(doc.Id)).Values<string>().ToList(); Assert.AreEqual(2, revisionIds.Count); Assert.IsTrue(revisionIds.Contains(rev4a.Id)); Assert.IsTrue(revisionIds.Contains(rev2b.Id)); } } Assert.IsTrue(foundRevsDiff); } }
public void TestContinuousReplicationErrorNotification() { if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"])) { Assert.Inconclusive("Replication tests disabled."); return; } var httpClientFactory = new MockHttpClientFactory(); manager.DefaultHttpClientFactory = httpClientFactory; var httpHandler = httpClientFactory.HttpHandler; httpHandler.AddResponderThrowExceptionAllRequests(); var pusher = database.CreatePushReplication(GetReplicationURL()); pusher.Continuous = true; var signal = new CountdownEvent(1); var observer = new ReplicationErrorObserver(signal); pusher.Changed += observer.Changed; pusher.Start(); var success = signal.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); pusher.Stop(); }
// Note that this should not happen anymore but this test will remain just to verify // the correct behavior if it does public void TestBulkGet404() { var factory = new MockHttpClientFactory(false); factory.HttpHandler.SetResponder("_changes", req => { var response = new HttpResponseMessage(HttpStatusCode.OK); response.Content = new StringContent(@"{""results"":[{""seq"":3,""id"":""somedoc"",""changes"": [{""rev"":""2-cafebabe""}]},{""seq"":4,""id"":""otherdoc"",""changes"":[{""rev"":""5-bedbedbe""}]}, {""seq"":5,""id"":""realdoc"",""changes"":[{""rev"":""1-12345abc""}]}]}"); return response; }); factory.HttpHandler.SetResponder("_bulk_get", req => { var response = new HttpResponseMessage(HttpStatusCode.OK); response.Content = new StringContent("--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\r\n" + "Content-Type: application/json; error=\"true\"\r\n\r\n" + "{\"error\":\"not_found\",\"id\":\"somedoc\",\"reason\":\"missing\",\"rev\":\"2-cafebabe\",\"status\":404}\r\n" + "--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\r\n" + "Content-Type: application/json; error=\"true\"\r\n\r\n" + "{\"error\":\"not_found\",\"id\":\"otherdoc\",\"reason\":\"missing\",\"rev\":\"5-bedbedbe\",\"status\":404}\r\n" + "--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\r\n" + "Content-Type: application/json\r\n\r\n" + "{\"_id\":\"realdoc\",\"_rev\":\"1-12345abc\",\"channels\":[\"unit_test\"],\"foo\":\"bar\"}\r\n" + "--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f--"); response.Content.Headers.Remove("Content-Type"); response.Content.Headers.TryAddWithoutValidation("Content-Type", "multipart/mixed; boundary=\"67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\""); return response; }); manager.DefaultHttpClientFactory = factory; using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var puller = database.CreatePullReplication(remoteDb.RemoteUri); RunReplication(puller); Assert.IsNotNull(puller.LastError); Assert.AreEqual(3, puller.ChangesCount); Assert.AreEqual(3, puller.CompletedChangesCount); Assert.AreEqual("5", puller.LastSequence); } }
public void TestHeaders() { if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"])) { Assert.Inconclusive("Replication tests disabled."); return; } var mockHttpClientFactory = new MockHttpClientFactory(); manager.DefaultHttpClientFactory = mockHttpClientFactory; var mockHttpHandler = mockHttpClientFactory.HttpHandler; mockHttpHandler.AddResponderThrowExceptionAllRequests(); using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var remote = remoteDb.RemoteUri; var puller = database.CreatePullReplication(remote); var headers = new Dictionary<string, string>(); headers["foo"] = "bar"; puller.Headers = headers; RunReplication(puller); Assert.IsNotNull(puller.LastError); var foundFooHeader = false; var requests = mockHttpHandler.CapturedRequests; foreach (var request in requests) { var requestHeaders = request.Headers.GetValues("foo"); foreach (var requestHeader in requestHeaders) { foundFooHeader = true; Assert.AreEqual("bar", requestHeader); } } Assert.IsTrue(foundFooHeader); } }
public void TestContinuousPushReplicationGoesIdle() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } // make sure we are starting empty Assert.AreEqual(0, database.GetLastSequenceNumber()); // add docs var properties1 = new Dictionary<string, object>(); properties1["doc1"] = "testContinuousPushReplicationGoesIdle"; CreateDocumentWithProperties(database, properties1); var httpClientFactory = new MockHttpClientFactory(); manager.DefaultHttpClientFactory = httpClientFactory; using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var firstPusher = database.CreatePushReplication(remoteDb.RemoteUri); firstPusher.Continuous = true; var checkpointId = firstPusher.RemoteCheckpointDocID(); var httpHandler = httpClientFactory.HttpHandler; MockHttpRequestHandler.HttpResponseDelegate localResponder = (request) => { var id = String.Format("_local/{0}", checkpointId); var json = "{" + String.Format("\"id\":\"{0}\",\"ok\":true,\"rev\":\"0-2\"", id) + "}"; return MockHttpRequestHandler.GenerateHttpResponseMessage(HttpStatusCode.Created, "OK", json); }; httpHandler.SetResponder("_local", localResponder); var replicationIdleSignal = new CountdownEvent(1); var replicationIdleObserver = new ReplicationIdleObserver(replicationIdleSignal); firstPusher.Changed += replicationIdleObserver.Changed; firstPusher.Start(); var success = replicationIdleSignal.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); StopReplication(firstPusher); // the last sequence should be "1" at this point. we will use this later var lastSequence = database.LastSequenceWithCheckpointId(checkpointId); Assert.AreEqual("1", lastSequence); // start a second continuous replication var secondPusher = database.CreatePushReplication(remoteDb.RemoteUri); secondPusher.Continuous = true; var secondPusherCheckpointId = secondPusher.RemoteCheckpointDocID(); Assert.AreEqual(checkpointId, secondPusherCheckpointId); // when this goes to fetch the checkpoint, return the last sequence from the previous replication localResponder = (request) => { var id = String.Format("_local/{0}", secondPusherCheckpointId); var json = String.Format("{{\"id\":\"{0}\",\"ok\":true,\"rev\":\"0-2\",\"lastSequence\":\"{1}\"}}", id, lastSequence); return MockHttpRequestHandler.GenerateHttpResponseMessage(HttpStatusCode.Created, "OK", json); }; httpHandler.SetResponder("_local", localResponder); // start second replication replicationIdleSignal = new CountdownEvent(1); replicationIdleObserver = new ReplicationIdleObserver(replicationIdleSignal); secondPusher.Changed += replicationIdleObserver.Changed; secondPusher.Start(); // wait until we get an IDLE event success = replicationIdleSignal.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(success); StopReplication(secondPusher); } }
public void TestPushReplicationCanMissDocs() { Assert.Inconclusive("Not sure this is a valid test."); if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"])) { Assert.Inconclusive("Replication tests disabled."); return; } Assert.AreEqual(0, database.GetLastSequenceNumber()); var properties1 = new Dictionary<string, object>(); properties1["doc1"] = "testPushReplicationCanMissDocs"; CreateDocumentWithProperties(database, properties1); var properties2 = new Dictionary<string, object>(); properties2["doc2"] = "testPushReplicationCanMissDocs"; var doc2 = CreateDocumentWithProperties(database, properties2); var doc2UnsavedRev = doc2.CreateRevision(); var attachmentStream = GetAsset("attachment.png"); doc2UnsavedRev.SetAttachment("attachment.png", "image/png", attachmentStream); var doc2Rev = doc2UnsavedRev.Save(); Assert.IsNotNull(doc2Rev); var httpClientFactory = new MockHttpClientFactory(); manager.DefaultHttpClientFactory = httpClientFactory; var httpHandler = httpClientFactory.HttpHandler; httpHandler.AddResponderFakeLocalDocumentUpdate404(); var json = "{\"error\":\"not_found\",\"reason\":\"missing\"}"; MockHttpRequestHandler.HttpResponseDelegate bulkDocsResponder = (request) => { return MockHttpRequestHandler.GenerateHttpResponseMessage(HttpStatusCode.NotFound, null, json); }; httpHandler.SetResponder("_bulk_docs", bulkDocsResponder); MockHttpRequestHandler.HttpResponseDelegate doc2Responder = (request) => { var responseObject = new Dictionary<string, object>(); responseObject["id"] = doc2.Id; responseObject["ok"] = true; responseObject["rev"] = doc2.CurrentRevisionId; return MockHttpRequestHandler.GenerateHttpResponseMessage(responseObject); }; httpHandler.SetResponder(doc2.Id, doc2Responder); var replicationDoneSignal = new CountdownEvent(1); var observer = new ReplicationObserver(replicationDoneSignal); var pusher = database.CreatePushReplication(GetReplicationURL()); pusher.Changed += observer.Changed; pusher.Start(); var success = replicationDoneSignal.Wait(TimeSpan.FromSeconds(5)); Assert.IsTrue(success); Assert.IsNotNull(pusher.LastError); Sleep(TimeSpan.FromMilliseconds(500)); var localLastSequence = database.LastSequenceWithCheckpointId(pusher.RemoteCheckpointDocID()); Log.D(Tag, "dtabase.lastSequenceWithCheckpointId(): " + localLastSequence); Log.D(Tag, "doc2.getCUrrentRevision().getSequence(): " + doc2.CurrentRevision.Sequence); // Since doc1 failed, the database should _not_ have had its lastSequence bumped to doc2's sequence number. // If it did, it's bug: github.com/couchbase/couchbase-lite-java-core/issues/95 Assert.IsFalse(doc2.CurrentRevision.Sequence.ToString().Equals(localLastSequence)); Assert.IsNull(localLastSequence); Assert.IsTrue(doc2.CurrentRevision.Sequence > 0); }
public void TestPusherBatching() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } // Create a bunch (InboxCapacity * 2) local documents var numDocsToSend = Replication.INBOX_CAPACITY * 2; for (var i = 0; i < numDocsToSend; i++) { var properties = new Dictionary<string, object>(); properties["testPusherBatching"] = i; var doc = database.CreateDocument(); var rev = doc.PutProperties(properties); Assert.IsNotNull(rev); } // Kick off a one time push replication to a mock var httpClientFactory = new MockHttpClientFactory(); var httpHandler = httpClientFactory.HttpHandler; httpHandler.AddResponderFakeLocalDocumentUpdate404(); manager.DefaultHttpClientFactory = httpClientFactory; using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var pusher = database.CreatePushReplication(remoteDb.RemoteUri); RunReplication(pusher); Assert.IsNull(pusher.LastError); var numDocsSent = 0; // Verify that only INBOX_SIZE documents are included in any given bulk post request var capturedRequests = httpHandler.CapturedRequests; foreach (var request in capturedRequests) { if (request.Method == HttpMethod.Post && request.RequestUri.AbsoluteUri.EndsWith("_bulk_docs", StringComparison.Ordinal)) { var bytes = request.Content.ReadAsByteArrayAsync().Result; var body = Manager.GetObjectMapper().ReadValue<IDictionary<string, object>>(bytes.AsEnumerable()); var docs = (JArray)body["docs"]; numDocsSent += docs.Count; } } Assert.AreEqual(numDocsToSend, numDocsSent); } }
public void TestBulkPullPermanentExceptionSurrender() { if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"])) { Assert.Inconclusive("Replication tests disabled."); return; } var fakeFactory = new MockHttpClientFactory(false); FlowControl flow = new FlowControl(new FlowItem[] { new ExceptionThrower(new TaskCanceledException()) { ExecutionCount = -1 }, }); fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) => flow.ExecuteNext<HttpResponseMessage>()); manager.DefaultHttpClientFactory = fakeFactory; ManagerOptions.Default.RequestTimeout = TimeSpan.FromSeconds(5); using (var remoteDb = _sg.CreateDatabase(TempDbName())) { CreatePullAndTest(20, remoteDb, repl => Assert.IsTrue(database.GetDocumentCount() < 20, "Somehow got all the docs")); } }
public void TestPushUpdatedDocWithoutReSendingAttachments() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } Assert.AreEqual(0, database.GetLastSequenceNumber()); var properties1 = new Dictionary<string, object>() { { "dynamic", 1 } }; var doc = CreateDocumentWithProperties(database, properties1); var rev1 = doc.CurrentRevision; var unsavedRev2 = doc.CreateRevision(); var attachmentStream = GetAsset("attachment.png"); unsavedRev2.SetAttachment("attachment.png", "image/png", attachmentStream); var rev2 = unsavedRev2.Save(); // Kick off a one time push replication to a mock var httpClientFactory = new MockHttpClientFactory(); var httpHandler = httpClientFactory.HttpHandler; httpHandler.AddResponderFakeLocalDocumentUpdate404(); httpHandler.SetResponder(doc.Id, (request) => { var content = new Dictionary<string, object>() { {"id", doc.Id}, {"ok", true}, {"rev", doc.CurrentRevisionId} }; return MockHttpRequestHandler.GenerateHttpResponseMessage(content); }); manager.DefaultHttpClientFactory = httpClientFactory; using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var pusher = database.CreatePushReplication(remoteDb.RemoteUri); RunReplication(pusher); httpHandler.ClearCapturedRequests(); var oldDoc = database.GetDocument(doc.Id); var unsavedRev = oldDoc.CreateRevision(); var props = new Dictionary<string, object>(oldDoc.UserProperties); props["dynamic"] = Convert.ToInt64(oldDoc.Properties["dynamic"]) + 1; unsavedRev.SetProperties(props); var savedRev = unsavedRev.Save(); httpHandler.SetResponder(doc.Id, (request) => { var content = new Dictionary<string, object>() { { "id", doc.Id }, { "ok", true }, { "rev", savedRev.Id } }; return MockHttpRequestHandler.GenerateHttpResponseMessage(content); }); httpHandler.SetResponder("_revs_diff", (request) => { var json = String.Format("{{\"{0}\":{{\"missing\":[\"{1}\"],\"possible_ancestors\":[\"{2},{3}\"]}}}}", doc.Id, savedRev.Id, rev1.Id, rev2.Id); return MockHttpRequestHandler.GenerateHttpResponseMessage(HttpStatusCode.OK, "OK", json); }); pusher = database.CreatePushReplication(remoteDb.RemoteUri); RunReplication(pusher); foreach (var request in httpHandler.CapturedRequests) { if (request.Method == HttpMethod.Put) { var isMultipartContent = (request.Content is MultipartContent); Assert.IsFalse(isMultipartContent); } } } }
public void TestPushPurgedDoc() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } var numBulkDocRequests = 0; HttpRequestMessage lastBulkDocsRequest = null; var doc = CreateDocumentWithProperties( database, new Dictionary<string, object> { {"testName", "testPurgeDocument"} } ); Assert.IsNotNull(doc); using (var remoteDb = _sg.CreateDatabase(TempDbName())) { var remote = remoteDb.RemoteUri; var factory = new MockHttpClientFactory(); factory.HttpHandler.ClearResponders(); factory.HttpHandler.AddResponderRevDiffsAllMissing(); factory.HttpHandler.AddResponderFakeLocalDocumentUpdate404(); factory.HttpHandler.AddResponderFakeBulkDocs(); manager.DefaultHttpClientFactory = factory; var pusher = database.CreatePushReplication(remote); var replicationCaughtUpSignal = new CountdownEvent(1); pusher.Changed += (sender, e) => { var changesCount = e.Source.ChangesCount; var completedChangesCount = e.Source.CompletedChangesCount; var msg = String.Format("changes: {0} completed changes: {1}", changesCount, completedChangesCount); WriteDebug(msg); if (changesCount > 0 && changesCount == completedChangesCount && replicationCaughtUpSignal.CurrentCount > 0) { replicationCaughtUpSignal.Signal(); } }; pusher.Start(); // wait until that doc is pushed var didNotTimeOut = replicationCaughtUpSignal.Wait(TimeSpan.FromSeconds(15)); Assert.IsTrue(didNotTimeOut); // at this point, we should have captured exactly 1 bulk docs request numBulkDocRequests = 0; var handler = factory.HttpHandler; foreach (var capturedRequest in handler.CapturedRequests) { if (capturedRequest.Method == HttpMethod.Post && capturedRequest.RequestUri.AbsoluteUri.EndsWith("_bulk_docs", StringComparison.Ordinal)) { lastBulkDocsRequest = capturedRequest; numBulkDocRequests++; } } Assert.AreEqual(1, numBulkDocRequests); // that bulk docs request should have the "start" key under its _revisions var jsonMap = MockHttpRequestHandler.GetJsonMapFromRequest(lastBulkDocsRequest); var docs = (jsonMap.Get("docs")).AsList<IDictionary<string,object>>(); var onlyDoc = docs[0]; var revisions = onlyDoc.Get("_revisions").AsDictionary<string,object>(); Assert.IsTrue(revisions.ContainsKey("start")); // Reset for the next attempt. handler.ClearCapturedRequests(); // now add a new revision, which will trigger the pusher to try to push it var properties = new Dictionary<string, object>(); properties["testName2"] = "update doc"; var unsavedRevision = doc.CreateRevision(); unsavedRevision.SetUserProperties(properties); unsavedRevision.Save(); // but then immediately purge it doc.Purge(); pusher.Start(); // wait for a while to give the replicator a chance to push it // (it should not actually push anything) Sleep(5 * 1000); // we should not have gotten any more _bulk_docs requests, because // the replicator should not have pushed anything else. // (in the case of the bug, it was trying to push the purged revision) numBulkDocRequests = 0; foreach (var capturedRequest in handler.CapturedRequests) { if (capturedRequest.Method == HttpMethod.Post && capturedRequest.RequestUri.AbsoluteUri.EndsWith("_bulk_docs", StringComparison.Ordinal)) { numBulkDocRequests++; } } Assert.AreEqual(0, numBulkDocRequests); pusher.Stop(); } }
public void TestBulkPullTransientExceptionRecovery() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } var fakeFactory = new MockHttpClientFactory(false); FlowControl flow = new FlowControl(new FlowItem[] { new FunctionRunner<HttpResponseMessage>(() => { Sleep(7000); return new RequestCorrectHttpMessage(); }) { ExecutionCount = 2 }, new FunctionRunner<HttpResponseMessage>(() => { fakeFactory.HttpHandler.ClearResponders(); return new RequestCorrectHttpMessage(); }) { ExecutionCount = 1 } }); fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) => flow.ExecuteNext<HttpResponseMessage>()); manager.DefaultHttpClientFactory = fakeFactory; #pragma warning disable 618 ManagerOptions.Default.RequestTimeout = TimeSpan.FromSeconds(5); #pragma warning restore 618 using (var remoteDb = _sg.CreateDatabase(TempDbName())) { CreatePullAndTest(20, remoteDb, (repl) => Assert.AreEqual(20, database.GetDocumentCount(), "Didn't recover from the error")); } Thread.Sleep(1000); }
public void TestRunReplicationWithError() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } var mockHttpClientFactory = new MockHttpClientFactory(); manager.DefaultHttpClientFactory = mockHttpClientFactory; var mockHttpHandler = mockHttpClientFactory.HttpHandler; mockHttpHandler.AddResponderFailAllRequests(HttpStatusCode.InternalServerError); var dbUrlString = "http://fake.test-url.com:4984/fake/"; var remote = new Uri(dbUrlString); var continuous = false; var r1 = new Pusher(database, remote, continuous, mockHttpClientFactory, new TaskFactory(new SingleTaskThreadpoolScheduler())); Assert.IsFalse(r1.Continuous); RunReplication(r1); Assert.AreEqual(ReplicationStatus.Stopped, r1.Status); Assert.AreEqual(0, r1.CompletedChangesCount); Assert.AreEqual(0, r1.ChangesCount); Assert.IsNotNull(r1.LastError); }
public void TestFailedBulkGetDoesntChangeLastSequence() { if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled"))) { Assert.Inconclusive("Replication tests disabled."); return; } string firstBulkGet = null; using(var remoteDb = _sg.CreateDatabase(TempDbName())) { var fakeFactory = new MockHttpClientFactory(false); fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) => { var str = default(string); if(request.Content is CompressedContent) { var stream = request.Content.ReadAsStreamAsync().Result; str = Encoding.UTF8.GetString(stream.ReadAllBytes()); } else { str = request.Content.ReadAsStringAsync().Result; } if (firstBulkGet == null || firstBulkGet.Equals(str)) { WriteDebug("Rejecting this bulk get because it looks like the first batch"); firstBulkGet = str; throw new OperationCanceledException(); } WriteDebug("Letting this bulk get through"); return new RequestCorrectHttpMessage(); }); int gotSequence = 0; fakeFactory.HttpHandler.SetResponder("doc", request => { Regex r = new Regex("doc[0-9]+"); var m = r.Match(request.RequestUri.PathAndQuery); if(m.Success) { var str = m.Captures[0].Value; var converted = Int32.Parse(str.Substring(3)) + 4; if(gotSequence == 0 || converted - gotSequence == 1) { gotSequence = converted; } } return new RequestCorrectHttpMessage(); }); manager.DefaultHttpClientFactory = fakeFactory; #pragma warning disable 618 Manager.DefaultOptions.MaxRevsToGetInBulk = 10; Manager.DefaultOptions.MaxOpenHttpConnections = 8; Manager.DefaultOptions.RequestTimeout = TimeSpan.FromSeconds(5); CreatePullAndTest((int)(Manager.DefaultOptions.MaxRevsToGetInBulk * 1.5), remoteDb, repl => { WriteDebug("Document count increased to {0} with last sequence '{1}'", database.GetDocumentCount(), repl.LastSequence); Assert.IsTrue(database.GetDocumentCount() > 0, "Didn't get docs from second bulk get batch"); Assert.AreEqual(gotSequence, Int32.Parse(repl.LastSequence), "LastSequence was advanced"); }); #pragma warning restore 618 Sleep(500); fakeFactory.HttpHandler.ClearResponders(); var pull = database.CreatePullReplication(remoteDb.RemoteUri); RunReplication(pull); Assert.AreEqual(pull.ChangesCount, pull.CompletedChangesCount); Assert.AreNotEqual(pull.LastSequence, "0"); } }
public void TestBulkPullTransientExceptionRecovery() { if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"])) { Assert.Inconclusive("Replication tests disabled."); return; } var initialRowCount = SyncGatewayRowCount(); var fakeFactory = new MockHttpClientFactory(false); FlowControl flow = new FlowControl(new FlowItem[] { new FunctionRunner<HttpResponseMessage>(() => { Thread.Sleep(7000); return new RequestCorrectHttpMessage(); }) { ExecutionCount = 2 }, new FunctionRunner<HttpResponseMessage>(() => { fakeFactory.HttpHandler.ClearResponders(); return new RequestCorrectHttpMessage(); }) { ExecutionCount = 1 } }); fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) => flow.ExecuteNext<HttpResponseMessage>()); manager.DefaultHttpClientFactory = fakeFactory; ManagerOptions.Default.RequestTimeout = TimeSpan.FromSeconds(5); CreatePullAndTest(20, (repl) => Assert.IsTrue(database.DocumentCount - initialRowCount == 20, "Didn't recover from the error")); }