public void TestFailedBulkGetDoesntChangeLastSequence()
{
if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"]))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var fakeFactory = new MockHttpClientFactory(false);
fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) =>
{
var str = request.Content.ReadAsStringAsync().Result;
if(str.IndexOf("doc2") != -1) {
Log.D(Tag, "Rejecting this bulk get because it looks like the first batch");
throw new OperationCanceledException();
}
Log.D(Tag, "Letting this bulk get through");
return new RequestCorrectHttpMessage();
});
manager.DefaultHttpClientFactory = fakeFactory;
Manager.DefaultOptions.MaxRevsToGetInBulk = 10;
Manager.DefaultOptions.MaxOpenHttpConnections = 8;
Manager.DefaultOptions.RequestTimeout = TimeSpan.FromSeconds(5);
CreatePullAndTest((int)(Manager.DefaultOptions.MaxRevsToGetInBulk * 1.5), repl => {
Log.D(Tag, "Document count increased to {0} with last sequence '{1}'", database.DocumentCount, repl.LastSequence);
Assert.IsTrue(database.DocumentCount > 0, "Didn't get docs from second bulk get batch");
Assert.AreEqual("0", repl.LastSequence, "LastSequence was advanced");
});
fakeFactory.HttpHandler.ClearResponders();
var pull = database.CreatePullReplication(GetReplicationURL());
RunReplication(pull);
Assert.AreEqual(pull.ChangesCount, pull.CompletedChangesCount);
Assert.AreNotEqual(pull.LastSequence, "0");
}
public void TestHeaders()
{
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var mockHttpClientFactory = new MockHttpClientFactory(false);
manager.DefaultHttpClientFactory = mockHttpClientFactory;
var mockHttpHandler = mockHttpClientFactory.HttpHandler;
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var remote = remoteDb.RemoteUri;
var puller = database.CreatePullReplication(remote);
var headers = new Dictionary<string, string>();
headers["foo"] = "bar";
var pusher = database.CreatePushReplication (remote);
pusher.Headers = headers;
pusher.Start ();
Sleep (5000);
pusher.Stop ();
ValidateHttpHeaders (mockHttpHandler);
mockHttpHandler.ClearCapturedRequests ();
puller.Headers = headers;
puller.Start();
Sleep(5000);
puller.Stop();
ValidateHttpHeaders (mockHttpHandler);
}
}
public void TestServerDoesNotSupportMultipart() {
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
Assert.AreEqual(0, database.GetLastSequenceNumber());
var properties1 = new Dictionary<string, object>() {
{ "dynamic", 1 }
};
var doc = CreateDocumentWithProperties(database, properties1);
var rev1 = doc.CurrentRevision;
var unsavedRev2 = doc.CreateRevision();
var attachmentStream = GetAsset("attachment.png");
unsavedRev2.SetAttachment("attachment.png", "image/png", attachmentStream);
var rev2 = unsavedRev2.Save();
attachmentStream.Dispose();
Assert.IsNotNull(rev2);
unsavedRev2.Dispose();
var httpClientFactory = new MockHttpClientFactory();
var httpHandler = httpClientFactory.HttpHandler;
httpHandler.AddResponderFakeLocalDocumentUpdate404();
var responders = new List<MockHttpRequestHandler.HttpResponseDelegate>();
responders.Add((request) =>
{
var json = "{\"error\":\"Unsupported Media Type\",\"reason\":\"missing\"}";
return MockHttpRequestHandler.GenerateHttpResponseMessage(HttpStatusCode.UnsupportedMediaType,
"Unsupported Media Type", json);
});
responders.Add((request) =>
{
var props = new Dictionary<string, object>()
{
{"id", doc.Id},
{"ok", true},
{"rev", doc.CurrentRevisionId}
};
return MockHttpRequestHandler.GenerateHttpResponseMessage(props);
});
MockHttpRequestHandler.HttpResponseDelegate chainResponder = (request) =>
{
if (responders.Count > 0) {
var responder = responders[0];
responders.RemoveAt(0);
return responder(request);
}
return null;
};
httpHandler.SetResponder(doc.Id, chainResponder);
manager.DefaultHttpClientFactory = httpClientFactory;
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var pusher = database.CreatePushReplication(remoteDb.RemoteUri);
RunReplication(pusher);
var count = 0;
foreach (var request in httpHandler.CapturedRequests) {
if (request.Method == HttpMethod.Put && request.RequestUri.PathAndQuery.Contains(doc.Id)) {
var isMultipartContent = (request.Content is MultipartContent);
if (count == 0) {
Assert.IsTrue(isMultipartContent);
}
else {
Assert.IsFalse(isMultipartContent);
}
count++;
}
}
}
}
public void TestBulkPullPermanentExceptionSurrender() {
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
Log.Domains.Sync.Level = Log.LogLevel.Debug;
var fakeFactory = new MockHttpClientFactory(false);
FlowControl flow = new FlowControl(new FlowItem[]
{
new ExceptionThrower(new SocketException()) { ExecutionCount = -1 },
});
fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) =>
flow.ExecuteNext<HttpResponseMessage>());
manager.DefaultHttpClientFactory = fakeFactory;
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
CreatePullAndTest(20, remoteDb, repl => Assert.IsTrue(database.GetDocumentCount() < 20, "Somehow got all the docs"));
}
}
public void TestContinuousReplicationErrorNotification() {
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var httpClientFactory = new MockHttpClientFactory();
manager.DefaultHttpClientFactory = httpClientFactory;
var httpHandler = httpClientFactory.HttpHandler;
httpHandler.AddResponderThrowExceptionAllRequests();
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var pusher = database.CreatePushReplication(remoteDb.RemoteUri);
pusher.Continuous = true;
var signal = new CountdownEvent(1);
var observer = new ReplicationErrorObserver(signal);
pusher.Changed += observer.Changed;
pusher.Start();
CreateDocuments(database, 10);
var success = signal.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
StopReplication(pusher);
}
}
private void RunPushReplicationWithTransientError(Int32 errorCode, string statusMessage, Boolean expectError)
{
var properties1 = new Dictionary<string, object>()
{
{"doc1", "testPushReplicationTransientError"}
};
CreateDocumentWithProperties(database, properties1);
var httpClientFactory = new MockHttpClientFactory(false);
var httpHandler = httpClientFactory.HttpHandler;
manager.DefaultHttpClientFactory = httpClientFactory;
MockHttpRequestHandler.HttpResponseDelegate sentinal = MockHttpRequestHandler.FakeBulkDocs;
var responders = new List<MockHttpRequestHandler.HttpResponseDelegate>();
responders.Add(MockHttpRequestHandler.TransientErrorResponder(errorCode, statusMessage));
MockHttpRequestHandler.HttpResponseDelegate chainResponder = (request) =>
{
if (responders.Count > 0) {
var responder = responders[0];
responders.RemoveAt(0);
return responder(request);
}
return sentinal(request);
};
httpHandler.SetResponder("_bulk_docs", chainResponder);
// Create a replication observer to wait until replication finishes
var replicationDoneSignal = new CountdownEvent(1);
var replicationFinishedObserver = new ReplicationObserver(replicationDoneSignal);
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var pusher = database.CreatePushReplication(remoteDb.RemoteUri);
pusher.Changed += replicationFinishedObserver.Changed;
// save the checkpoint id for later usage
var checkpointId = pusher.RemoteCheckpointDocID();
// kick off the replication
pusher.Start();
// wait for it to finish
var success = replicationDoneSignal.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
if (expectError) {
Assert.IsNotNull(pusher.LastError);
}
else {
Assert.IsNull(pusher.LastError);
}
// workaround for the fact that the replicationDoneSignal.Await() call could unblock before all
// the statements in Replication.Stopped() have even had a chance to execute.
Sleep(500);
var localLastSequence = database.LastSequenceWithCheckpointId(checkpointId);
if (expectError) {
Assert.Null(localLastSequence);
}
else {
Assert.IsNotNull(localLastSequence);
}
}
}
public void TestAllLeafRevisionsArePushed()
{
var httpClientFactory = new MockHttpClientFactory();
var httpHandler = httpClientFactory.HttpHandler;
httpHandler.AddResponderRevDiffsAllMissing();
httpHandler.AddResponderFakeLocalDocumentUpdate404();
httpHandler.ResponseDelayMilliseconds = 250;
manager.DefaultHttpClientFactory = httpClientFactory;
var doc = database.CreateDocument();
var rev1a = doc.CreateRevision().Save();
var rev2a = rev1a.CreateRevision().Save();
var rev3a = rev2a.CreateRevision().Save();
// delete the branch we've been using, then create a new one to replace it
var rev4a = rev3a.DeleteDocument();
var rev2b = rev1a.CreateRevision().Save(true);
Assert.AreEqual(rev2b.Id, doc.CurrentRevisionId);
// sync with remote DB -- should push both leaf revisions
var pusher = database.CreatePushReplication(GetReplicationURL());
RunReplication(pusher);
Assert.IsNull(pusher.LastError);
var foundRevsDiff = false;
var capturedRequests = httpHandler.CapturedRequests;
foreach (var httpRequest in capturedRequests)
{
var uriString = httpRequest.RequestUri.ToString();
if (uriString.EndsWith("_revs_diff", StringComparison.Ordinal))
{
foundRevsDiff = true;
var jsonMap = MockHttpRequestHandler.GetJsonMapFromRequest(httpRequest);
var revisionIds = ((JArray)jsonMap.Get(doc.Id)).Values<string>().ToList();
Assert.AreEqual(2, revisionIds.Count);
Assert.IsTrue(revisionIds.Contains(rev4a.Id));
Assert.IsTrue(revisionIds.Contains(rev2b.Id));
}
}
Assert.IsTrue(foundRevsDiff);
}
[Test] // This test takes nearly 5 minutes to run, so only run when needed
#endif
public void TestLongRemovedChangesFeed()
{
var random = new Random();
var changesFeed = new StringBuilder("{\"results\":[");
const int limit = 100000;
HashSet<string> removedIDSet = new HashSet<string>();
for (var i = 1; i < limit; i++) {
var removed = random.NextDouble() >= 0.5;
if (removed) {
var removedID = Misc.CreateGUID();
changesFeed.AppendFormat("{{\"seq\":\"{0}\",\"id\":\"{1}\",\"removed\":[\"fake\"]," +
"\"changes\":[{{\"rev\":\"1-deadbeef\"}}]}},",
i, removedID);
removedIDSet.Add(removedID);
} else {
changesFeed.AppendFormat("{{\"seq\":\"{0}\",\"id\":\"{1}\",\"changes\":[{{\"rev\":\"1-deadbeef\"}}]}},",
i, Misc.CreateGUID());
}
}
changesFeed.AppendFormat("{{\"seq\":\"{0}\",\"id\":\"{1}\",\"changes\":[{{\"rev\":\"1-deadbeef\"}}]}}]," +
"last_seq: \"{0}\"}}",
limit, Misc.CreateGUID());
var factory = new MockHttpClientFactory(false);
factory.HttpHandler.SetResponder("_changes", req =>
{
var response = new HttpResponseMessage(HttpStatusCode.OK);
var changesString = changesFeed.ToString();
response.Content = new StringContent(changesString);
return response;
});
factory.HttpHandler.SetResponder("_bulk_get", req =>
{
var contentStream = req.Content.ReadAsStreamAsync().Result;
var content = Manager.GetObjectMapper().ReadValue<IDictionary<string, object>>(contentStream);
var responseBody = new StringBuilder("--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\r\n");
foreach(var obj in content["docs"] as IEnumerable) {
var dict = obj.AsDictionary<string, object>();
var nonexistent = removedIDSet.Contains(dict.GetCast<string>("id"));
if(nonexistent) {
return new HttpResponseMessage(HttpStatusCode.InternalServerError); // Just so we can know
} else {
responseBody.Append("Content-Type: application/json\r\n\r\n");
responseBody.AppendFormat("{{\"_id\":\"{0}\",\"_rev\":\"1-deadbeef\",\"foo\":\"bar\"}}\r\n", dict["id"]);
}
responseBody.Append("--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\r\n");
}
responseBody.Remove(responseBody.Length - 2, 2);
responseBody.Append("--");
var retVal = new HttpResponseMessage(HttpStatusCode.OK);
var responseString = responseBody.ToString();
retVal.Content = new StringContent(responseString);
retVal.Content.Headers.Remove("Content-Type");
retVal.Content.Headers.TryAddWithoutValidation("Content-Type", "multipart/mixed; boundary=\"67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\"");
return retVal;
});
manager.DefaultHttpClientFactory = factory;
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var puller = database.CreatePullReplication(remoteDb.RemoteUri);
RunReplication(puller);
Assert.AreEqual(ReplicationStatus.Stopped, puller.Status);
Assert.AreNotEqual(limit, puller.ChangesCount);
Assert.AreNotEqual(limit, puller.CompletedChangesCount);
Assert.AreEqual(limit.ToString(), puller.LastSequence);
}
Sleep(1000);
}
public ChangeTrackerTestClient(CountDownLatch stoppedSignal, CountDownLatch changedSignal)
{
this.stoppedSignal = stoppedSignal;
this.changedSignal = changedSignal;
HttpClientFactory = new MockHttpClientFactory();
}
public void TestHeaders()
{
var mockHttpClientFactory = new MockHttpClientFactory();
manager.DefaultHttpClientFactory = mockHttpClientFactory;
var mockHttpHandler = mockHttpClientFactory.HttpHandler;
mockHttpHandler.AddResponderThrowExceptionAllRequests();
Uri remote = GetReplicationURL();
Replication puller = database.CreatePullReplication(remote);
var headers = new Dictionary<string, string>();
headers["foo"] = "bar";
puller.Headers = headers;
RunReplication(puller);
Assert.IsNotNull(puller.LastError);
var foundFooHeader = false;
var requests = mockHttpHandler.CapturedRequests;
foreach (var request in requests)
{
var requestHeaders = request.Headers.GetValues("foo");
foreach (var requestHeader in requestHeaders)
{
foundFooHeader = true;
Assert.AreEqual("bar", requestHeader);
}
}
Assert.IsTrue(foundFooHeader);
}
public void TestChangeTrackerBackoffInvalidJson()
{
var factory = new MockHttpClientFactory();
var httpHandler = (MockHttpRequestHandler)factory.HttpHandler;
httpHandler.AddResponderReturnInvalidChangesFeedJson();
TestChangeTrackerBackoff(factory);
}
public void TestChangeTrackerBackoffExceptions()
{
var factory = new MockHttpClientFactory();
var httpHandler = (MockHttpRequestHandler)factory.HttpHandler;
httpHandler.AddResponderThrowExceptionAllRequests();
TestChangeTrackerBackoff(factory);
}
private void TestChangeTrackerBackoff(MockHttpClientFactory httpClientFactory)
{
var changeTrackerFinishedSignal = new CountDownLatch(1);
var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, null);
client.HttpClientFactory = httpClientFactory;
var testUrl = GetReplicationURL();
var scheduler = new SingleTaskThreadpoolScheduler();
var changeTracker = new ChangeTracker(testUrl, ChangeTrackerMode.LongPoll, 0, false, client, new TaskFactory(scheduler));
changeTracker.UsePost = IsSyncGateway(testUrl);
changeTracker.Start();
// sleep for a few seconds
Thread.Sleep(15 * 1000);
// make sure we got less than 10 requests in those 10 seconds (if it was hammering, we'd get a lot more)
var handler = client.HttpRequestHandler;
Assert.IsTrue(handler.CapturedRequests.Count < 25);
Assert.IsTrue(changeTracker.backoff.NumAttempts > 0, "Observed attempts: {0}".Fmt(changeTracker.backoff.NumAttempts));
handler.ClearResponders();
handler.AddResponderReturnEmptyChangesFeed();
// at this point, the change tracker backoff should cause it to sleep for about 3 seconds
// and so lets wait 3 seconds until it wakes up and starts getting valid responses
Thread.Sleep(3 * 1000);
// now find the delta in requests received in a 2s period
int before = handler.CapturedRequests.Count;
Thread.Sleep(2 * 1000);
int after = handler.CapturedRequests.Count;
// assert that the delta is high, because at this point the change tracker should
// be hammering away
Assert.IsTrue((after - before) > 25);
// the backoff numAttempts should have been reset to 0
Assert.IsTrue(changeTracker.backoff.NumAttempts == 0);
changeTracker.Stop();
var success = changeTrackerFinishedSignal.Await(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
}
public void TestAllLeafRevisionsArePushed()
{
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var httpClientFactory = new MockHttpClientFactory();
var httpHandler = httpClientFactory.HttpHandler;
httpHandler.AddResponderRevDiffsAllMissing();
httpHandler.AddResponderFakeLocalDocumentUpdate404();
httpHandler.ResponseDelayMilliseconds = 250;
manager.DefaultHttpClientFactory = httpClientFactory;
var doc = database.CreateDocument();
var rev1a = doc.CreateRevision().Save();
var rev2a = rev1a.CreateRevision().Save();
var rev3a = rev2a.CreateRevision().Save();
// delete the branch we've been using, then create a new one to replace it
var rev4a = rev3a.DeleteDocument();
var unsaved = rev1a.CreateRevision();
unsaved.SetUserProperties(new Dictionary<string, object> { { "foo", "bar" } });
var rev2b = unsaved.Save(true);
Assert.AreEqual(rev2b.Id, doc.CurrentRevisionId);
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
// sync with remote DB -- should push both leaf revisions
var pusher = database.CreatePushReplication(remoteDb.RemoteUri);
RunReplication(pusher);
Assert.IsNull(pusher.LastError);
var foundRevsDiff = false;
var capturedRequests = httpHandler.CapturedRequests;
foreach (var httpRequest in capturedRequests) {
var uriString = httpRequest.RequestUri.ToString();
if (uriString.EndsWith("_revs_diff", StringComparison.Ordinal)) {
foundRevsDiff = true;
var jsonMap = MockHttpRequestHandler.GetJsonMapFromRequest(httpRequest);
var revisionIds = ((JArray)jsonMap.Get(doc.Id)).Values<string>().ToList();
Assert.AreEqual(2, revisionIds.Count);
Assert.IsTrue(revisionIds.Contains(rev4a.Id));
Assert.IsTrue(revisionIds.Contains(rev2b.Id));
}
}
Assert.IsTrue(foundRevsDiff);
}
}
public void TestContinuousReplicationErrorNotification() {
if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"]))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var httpClientFactory = new MockHttpClientFactory();
manager.DefaultHttpClientFactory = httpClientFactory;
var httpHandler = httpClientFactory.HttpHandler;
httpHandler.AddResponderThrowExceptionAllRequests();
var pusher = database.CreatePushReplication(GetReplicationURL());
pusher.Continuous = true;
var signal = new CountdownEvent(1);
var observer = new ReplicationErrorObserver(signal);
pusher.Changed += observer.Changed;
pusher.Start();
var success = signal.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
pusher.Stop();
}
// Note that this should not happen anymore but this test will remain just to verify
// the correct behavior if it does
public void TestBulkGet404()
{
var factory = new MockHttpClientFactory(false);
factory.HttpHandler.SetResponder("_changes", req =>
{
var response = new HttpResponseMessage(HttpStatusCode.OK);
response.Content = new StringContent(@"{""results"":[{""seq"":3,""id"":""somedoc"",""changes"":
[{""rev"":""2-cafebabe""}]},{""seq"":4,""id"":""otherdoc"",""changes"":[{""rev"":""5-bedbedbe""}]},
{""seq"":5,""id"":""realdoc"",""changes"":[{""rev"":""1-12345abc""}]}]}");
return response;
});
factory.HttpHandler.SetResponder("_bulk_get", req =>
{
var response = new HttpResponseMessage(HttpStatusCode.OK);
response.Content = new StringContent("--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\r\n" +
"Content-Type: application/json; error=\"true\"\r\n\r\n" +
"{\"error\":\"not_found\",\"id\":\"somedoc\",\"reason\":\"missing\",\"rev\":\"2-cafebabe\",\"status\":404}\r\n" +
"--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\r\n" +
"Content-Type: application/json; error=\"true\"\r\n\r\n" +
"{\"error\":\"not_found\",\"id\":\"otherdoc\",\"reason\":\"missing\",\"rev\":\"5-bedbedbe\",\"status\":404}\r\n" +
"--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\r\n" +
"Content-Type: application/json\r\n\r\n" +
"{\"_id\":\"realdoc\",\"_rev\":\"1-12345abc\",\"channels\":[\"unit_test\"],\"foo\":\"bar\"}\r\n" +
"--67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f--");
response.Content.Headers.Remove("Content-Type");
response.Content.Headers.TryAddWithoutValidation("Content-Type", "multipart/mixed; boundary=\"67aac1bcad803590b9a9e1999fc539438b3363fab35a24c17990188b222f\"");
return response;
});
manager.DefaultHttpClientFactory = factory;
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var puller = database.CreatePullReplication(remoteDb.RemoteUri);
RunReplication(puller);
Assert.IsNotNull(puller.LastError);
Assert.AreEqual(3, puller.ChangesCount);
Assert.AreEqual(3, puller.CompletedChangesCount);
Assert.AreEqual("5", puller.LastSequence);
}
}
public void TestHeaders()
{
if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"]))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var mockHttpClientFactory = new MockHttpClientFactory();
manager.DefaultHttpClientFactory = mockHttpClientFactory;
var mockHttpHandler = mockHttpClientFactory.HttpHandler;
mockHttpHandler.AddResponderThrowExceptionAllRequests();
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var remote = remoteDb.RemoteUri;
var puller = database.CreatePullReplication(remote);
var headers = new Dictionary<string, string>();
headers["foo"] = "bar";
puller.Headers = headers;
RunReplication(puller);
Assert.IsNotNull(puller.LastError);
var foundFooHeader = false;
var requests = mockHttpHandler.CapturedRequests;
foreach (var request in requests) {
var requestHeaders = request.Headers.GetValues("foo");
foreach (var requestHeader in requestHeaders) {
foundFooHeader = true;
Assert.AreEqual("bar", requestHeader);
}
}
Assert.IsTrue(foundFooHeader);
}
}
public void TestContinuousPushReplicationGoesIdle()
{
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
// make sure we are starting empty
Assert.AreEqual(0, database.GetLastSequenceNumber());
// add docs
var properties1 = new Dictionary<string, object>();
properties1["doc1"] = "testContinuousPushReplicationGoesIdle";
CreateDocumentWithProperties(database, properties1);
var httpClientFactory = new MockHttpClientFactory();
manager.DefaultHttpClientFactory = httpClientFactory;
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var firstPusher = database.CreatePushReplication(remoteDb.RemoteUri);
firstPusher.Continuous = true;
var checkpointId = firstPusher.RemoteCheckpointDocID();
var httpHandler = httpClientFactory.HttpHandler;
MockHttpRequestHandler.HttpResponseDelegate localResponder = (request) =>
{
var id = String.Format("_local/{0}", checkpointId);
var json = "{" + String.Format("\"id\":\"{0}\",\"ok\":true,\"rev\":\"0-2\"", id) + "}";
return MockHttpRequestHandler.GenerateHttpResponseMessage(HttpStatusCode.Created, "OK", json);
};
httpHandler.SetResponder("_local", localResponder);
var replicationIdleSignal = new CountdownEvent(1);
var replicationIdleObserver = new ReplicationIdleObserver(replicationIdleSignal);
firstPusher.Changed += replicationIdleObserver.Changed;
firstPusher.Start();
var success = replicationIdleSignal.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
StopReplication(firstPusher);
// the last sequence should be "1" at this point. we will use this later
var lastSequence = database.LastSequenceWithCheckpointId(checkpointId);
Assert.AreEqual("1", lastSequence);
// start a second continuous replication
var secondPusher = database.CreatePushReplication(remoteDb.RemoteUri);
secondPusher.Continuous = true;
var secondPusherCheckpointId = secondPusher.RemoteCheckpointDocID();
Assert.AreEqual(checkpointId, secondPusherCheckpointId);
// when this goes to fetch the checkpoint, return the last sequence from the previous replication
localResponder = (request) =>
{
var id = String.Format("_local/{0}", secondPusherCheckpointId);
var json = String.Format("{{\"id\":\"{0}\",\"ok\":true,\"rev\":\"0-2\",\"lastSequence\":\"{1}\"}}", id, lastSequence);
return MockHttpRequestHandler.GenerateHttpResponseMessage(HttpStatusCode.Created, "OK", json);
};
httpHandler.SetResponder("_local", localResponder);
// start second replication
replicationIdleSignal = new CountdownEvent(1);
replicationIdleObserver = new ReplicationIdleObserver(replicationIdleSignal);
secondPusher.Changed += replicationIdleObserver.Changed;
secondPusher.Start();
// wait until we get an IDLE event
success = replicationIdleSignal.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
StopReplication(secondPusher);
}
}
public void TestPushReplicationCanMissDocs()
{
Assert.Inconclusive("Not sure this is a valid test.");
if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"]))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
Assert.AreEqual(0, database.GetLastSequenceNumber());
var properties1 = new Dictionary<string, object>();
properties1["doc1"] = "testPushReplicationCanMissDocs";
CreateDocumentWithProperties(database, properties1);
var properties2 = new Dictionary<string, object>();
properties2["doc2"] = "testPushReplicationCanMissDocs";
var doc2 = CreateDocumentWithProperties(database, properties2);
var doc2UnsavedRev = doc2.CreateRevision();
var attachmentStream = GetAsset("attachment.png");
doc2UnsavedRev.SetAttachment("attachment.png", "image/png", attachmentStream);
var doc2Rev = doc2UnsavedRev.Save();
Assert.IsNotNull(doc2Rev);
var httpClientFactory = new MockHttpClientFactory();
manager.DefaultHttpClientFactory = httpClientFactory;
var httpHandler = httpClientFactory.HttpHandler;
httpHandler.AddResponderFakeLocalDocumentUpdate404();
var json = "{\"error\":\"not_found\",\"reason\":\"missing\"}";
MockHttpRequestHandler.HttpResponseDelegate bulkDocsResponder = (request) =>
{
return MockHttpRequestHandler.GenerateHttpResponseMessage(HttpStatusCode.NotFound, null, json);
};
httpHandler.SetResponder("_bulk_docs", bulkDocsResponder);
MockHttpRequestHandler.HttpResponseDelegate doc2Responder = (request) =>
{
var responseObject = new Dictionary<string, object>();
responseObject["id"] = doc2.Id;
responseObject["ok"] = true;
responseObject["rev"] = doc2.CurrentRevisionId;
return MockHttpRequestHandler.GenerateHttpResponseMessage(responseObject);
};
httpHandler.SetResponder(doc2.Id, doc2Responder);
var replicationDoneSignal = new CountdownEvent(1);
var observer = new ReplicationObserver(replicationDoneSignal);
var pusher = database.CreatePushReplication(GetReplicationURL());
pusher.Changed += observer.Changed;
pusher.Start();
var success = replicationDoneSignal.Wait(TimeSpan.FromSeconds(5));
Assert.IsTrue(success);
Assert.IsNotNull(pusher.LastError);
Sleep(TimeSpan.FromMilliseconds(500));
var localLastSequence = database.LastSequenceWithCheckpointId(pusher.RemoteCheckpointDocID());
Log.D(Tag, "dtabase.lastSequenceWithCheckpointId(): " + localLastSequence);
Log.D(Tag, "doc2.getCUrrentRevision().getSequence(): " + doc2.CurrentRevision.Sequence);
// Since doc1 failed, the database should _not_ have had its lastSequence bumped to doc2's sequence number.
// If it did, it's bug: github.com/couchbase/couchbase-lite-java-core/issues/95
Assert.IsFalse(doc2.CurrentRevision.Sequence.ToString().Equals(localLastSequence));
Assert.IsNull(localLastSequence);
Assert.IsTrue(doc2.CurrentRevision.Sequence > 0);
}
public void TestPusherBatching()
{
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
// Create a bunch (InboxCapacity * 2) local documents
var numDocsToSend = Replication.INBOX_CAPACITY * 2;
for (var i = 0; i < numDocsToSend; i++)
{
var properties = new Dictionary<string, object>();
properties["testPusherBatching"] = i;
var doc = database.CreateDocument();
var rev = doc.PutProperties(properties);
Assert.IsNotNull(rev);
}
// Kick off a one time push replication to a mock
var httpClientFactory = new MockHttpClientFactory();
var httpHandler = httpClientFactory.HttpHandler;
httpHandler.AddResponderFakeLocalDocumentUpdate404();
manager.DefaultHttpClientFactory = httpClientFactory;
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var pusher = database.CreatePushReplication(remoteDb.RemoteUri);
RunReplication(pusher);
Assert.IsNull(pusher.LastError);
var numDocsSent = 0;
// Verify that only INBOX_SIZE documents are included in any given bulk post request
var capturedRequests = httpHandler.CapturedRequests;
foreach (var request in capturedRequests) {
if (request.Method == HttpMethod.Post &&
request.RequestUri.AbsoluteUri.EndsWith("_bulk_docs", StringComparison.Ordinal)) {
var bytes = request.Content.ReadAsByteArrayAsync().Result;
var body = Manager.GetObjectMapper().ReadValue<IDictionary<string, object>>(bytes.AsEnumerable());
var docs = (JArray)body["docs"];
numDocsSent += docs.Count;
}
}
Assert.AreEqual(numDocsToSend, numDocsSent);
}
}
public void TestBulkPullPermanentExceptionSurrender() {
if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"]))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var fakeFactory = new MockHttpClientFactory(false);
FlowControl flow = new FlowControl(new FlowItem[]
{
new ExceptionThrower(new TaskCanceledException()) { ExecutionCount = -1 },
});
fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) =>
flow.ExecuteNext<HttpResponseMessage>());
manager.DefaultHttpClientFactory = fakeFactory;
ManagerOptions.Default.RequestTimeout = TimeSpan.FromSeconds(5);
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
CreatePullAndTest(20, remoteDb, repl => Assert.IsTrue(database.GetDocumentCount() < 20, "Somehow got all the docs"));
}
}
public void TestPushUpdatedDocWithoutReSendingAttachments()
{
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
Assert.AreEqual(0, database.GetLastSequenceNumber());
var properties1 = new Dictionary<string, object>() {
{ "dynamic", 1 }
};
var doc = CreateDocumentWithProperties(database, properties1);
var rev1 = doc.CurrentRevision;
var unsavedRev2 = doc.CreateRevision();
var attachmentStream = GetAsset("attachment.png");
unsavedRev2.SetAttachment("attachment.png", "image/png", attachmentStream);
var rev2 = unsavedRev2.Save();
// Kick off a one time push replication to a mock
var httpClientFactory = new MockHttpClientFactory();
var httpHandler = httpClientFactory.HttpHandler;
httpHandler.AddResponderFakeLocalDocumentUpdate404();
httpHandler.SetResponder(doc.Id, (request) =>
{
var content = new Dictionary<string, object>()
{
{"id", doc.Id},
{"ok", true},
{"rev", doc.CurrentRevisionId}
};
return MockHttpRequestHandler.GenerateHttpResponseMessage(content);
});
manager.DefaultHttpClientFactory = httpClientFactory;
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var pusher = database.CreatePushReplication(remoteDb.RemoteUri);
RunReplication(pusher);
httpHandler.ClearCapturedRequests();
var oldDoc = database.GetDocument(doc.Id);
var unsavedRev = oldDoc.CreateRevision();
var props = new Dictionary<string, object>(oldDoc.UserProperties);
props["dynamic"] = Convert.ToInt64(oldDoc.Properties["dynamic"]) + 1;
unsavedRev.SetProperties(props);
var savedRev = unsavedRev.Save();
httpHandler.SetResponder(doc.Id, (request) =>
{
var content = new Dictionary<string, object>() {
{ "id", doc.Id },
{ "ok", true },
{ "rev", savedRev.Id }
};
return MockHttpRequestHandler.GenerateHttpResponseMessage(content);
});
httpHandler.SetResponder("_revs_diff", (request) =>
{
var json = String.Format("{{\"{0}\":{{\"missing\":[\"{1}\"],\"possible_ancestors\":[\"{2},{3}\"]}}}}", doc.Id, savedRev.Id, rev1.Id, rev2.Id);
return MockHttpRequestHandler.GenerateHttpResponseMessage(HttpStatusCode.OK, "OK", json);
});
pusher = database.CreatePushReplication(remoteDb.RemoteUri);
RunReplication(pusher);
foreach (var request in httpHandler.CapturedRequests) {
if (request.Method == HttpMethod.Put) {
var isMultipartContent = (request.Content is MultipartContent);
Assert.IsFalse(isMultipartContent);
}
}
}
}
public void TestPushPurgedDoc()
{
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var numBulkDocRequests = 0;
HttpRequestMessage lastBulkDocsRequest = null;
var doc = CreateDocumentWithProperties(
database,
new Dictionary<string, object>
{
{"testName", "testPurgeDocument"}
}
);
Assert.IsNotNull(doc);
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
var remote = remoteDb.RemoteUri;
var factory = new MockHttpClientFactory();
factory.HttpHandler.ClearResponders();
factory.HttpHandler.AddResponderRevDiffsAllMissing();
factory.HttpHandler.AddResponderFakeLocalDocumentUpdate404();
factory.HttpHandler.AddResponderFakeBulkDocs();
manager.DefaultHttpClientFactory = factory;
var pusher = database.CreatePushReplication(remote);
var replicationCaughtUpSignal = new CountdownEvent(1);
pusher.Changed += (sender, e) =>
{
var changesCount = e.Source.ChangesCount;
var completedChangesCount = e.Source.CompletedChangesCount;
var msg = String.Format("changes: {0} completed changes: {1}", changesCount, completedChangesCount);
WriteDebug(msg);
if (changesCount > 0 && changesCount == completedChangesCount
&& replicationCaughtUpSignal.CurrentCount > 0) {
replicationCaughtUpSignal.Signal();
}
};
pusher.Start();
// wait until that doc is pushed
var didNotTimeOut = replicationCaughtUpSignal.Wait(TimeSpan.FromSeconds(15));
Assert.IsTrue(didNotTimeOut);
// at this point, we should have captured exactly 1 bulk docs request
numBulkDocRequests = 0;
var handler = factory.HttpHandler;
foreach (var capturedRequest in handler.CapturedRequests) {
if (capturedRequest.Method == HttpMethod.Post && capturedRequest.RequestUri.AbsoluteUri.EndsWith("_bulk_docs", StringComparison.Ordinal)) {
lastBulkDocsRequest = capturedRequest;
numBulkDocRequests++;
}
}
Assert.AreEqual(1, numBulkDocRequests);
// that bulk docs request should have the "start" key under its _revisions
var jsonMap = MockHttpRequestHandler.GetJsonMapFromRequest(lastBulkDocsRequest);
var docs = (jsonMap.Get("docs")).AsList<IDictionary<string,object>>();
var onlyDoc = docs[0];
var revisions = onlyDoc.Get("_revisions").AsDictionary<string,object>();
Assert.IsTrue(revisions.ContainsKey("start"));
// Reset for the next attempt.
handler.ClearCapturedRequests();
// now add a new revision, which will trigger the pusher to try to push it
var properties = new Dictionary<string, object>();
properties["testName2"] = "update doc";
var unsavedRevision = doc.CreateRevision();
unsavedRevision.SetUserProperties(properties);
unsavedRevision.Save();
// but then immediately purge it
doc.Purge();
pusher.Start();
// wait for a while to give the replicator a chance to push it
// (it should not actually push anything)
Sleep(5 * 1000);
// we should not have gotten any more _bulk_docs requests, because
// the replicator should not have pushed anything else.
// (in the case of the bug, it was trying to push the purged revision)
numBulkDocRequests = 0;
foreach (var capturedRequest in handler.CapturedRequests) {
if (capturedRequest.Method == HttpMethod.Post && capturedRequest.RequestUri.AbsoluteUri.EndsWith("_bulk_docs", StringComparison.Ordinal)) {
numBulkDocRequests++;
}
}
Assert.AreEqual(0, numBulkDocRequests);
pusher.Stop();
}
}
public void TestBulkPullTransientExceptionRecovery() {
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var fakeFactory = new MockHttpClientFactory(false);
FlowControl flow = new FlowControl(new FlowItem[]
{
new FunctionRunner<HttpResponseMessage>(() => {
Sleep(7000);
return new RequestCorrectHttpMessage();
}) { ExecutionCount = 2 },
new FunctionRunner<HttpResponseMessage>(() => {
fakeFactory.HttpHandler.ClearResponders();
return new RequestCorrectHttpMessage();
}) { ExecutionCount = 1 }
});
fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) =>
flow.ExecuteNext<HttpResponseMessage>());
manager.DefaultHttpClientFactory = fakeFactory;
#pragma warning disable 618
ManagerOptions.Default.RequestTimeout = TimeSpan.FromSeconds(5);
#pragma warning restore 618
using (var remoteDb = _sg.CreateDatabase(TempDbName())) {
CreatePullAndTest(20, remoteDb, (repl) => Assert.AreEqual(20, database.GetDocumentCount(), "Didn't recover from the error"));
}
Thread.Sleep(1000);
}
public void TestRunReplicationWithError()
{
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var mockHttpClientFactory = new MockHttpClientFactory();
manager.DefaultHttpClientFactory = mockHttpClientFactory;
var mockHttpHandler = mockHttpClientFactory.HttpHandler;
mockHttpHandler.AddResponderFailAllRequests(HttpStatusCode.InternalServerError);
var dbUrlString = "http://fake.test-url.com:4984/fake/";
var remote = new Uri(dbUrlString);
var continuous = false;
var r1 = new Pusher(database, remote, continuous, mockHttpClientFactory, new TaskFactory(new SingleTaskThreadpoolScheduler()));
Assert.IsFalse(r1.Continuous);
RunReplication(r1);
Assert.AreEqual(ReplicationStatus.Stopped, r1.Status);
Assert.AreEqual(0, r1.CompletedChangesCount);
Assert.AreEqual(0, r1.ChangesCount);
Assert.IsNotNull(r1.LastError);
}
public void TestFailedBulkGetDoesntChangeLastSequence()
{
if (!Boolean.Parse((string)GetProperty("replicationTestsEnabled")))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
string firstBulkGet = null;
using(var remoteDb = _sg.CreateDatabase(TempDbName())) {
var fakeFactory = new MockHttpClientFactory(false);
fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) =>
{
var str = default(string);
if(request.Content is CompressedContent) {
var stream = request.Content.ReadAsStreamAsync().Result;
str = Encoding.UTF8.GetString(stream.ReadAllBytes());
} else {
str = request.Content.ReadAsStringAsync().Result;
}
if (firstBulkGet == null || firstBulkGet.Equals(str)) {
WriteDebug("Rejecting this bulk get because it looks like the first batch");
firstBulkGet = str;
throw new OperationCanceledException();
}
WriteDebug("Letting this bulk get through");
return new RequestCorrectHttpMessage();
});
int gotSequence = 0;
fakeFactory.HttpHandler.SetResponder("doc", request =>
{
Regex r = new Regex("doc[0-9]+");
var m = r.Match(request.RequestUri.PathAndQuery);
if(m.Success) {
var str = m.Captures[0].Value;
var converted = Int32.Parse(str.Substring(3)) + 4;
if(gotSequence == 0 || converted - gotSequence == 1) {
gotSequence = converted;
}
}
return new RequestCorrectHttpMessage();
});
manager.DefaultHttpClientFactory = fakeFactory;
#pragma warning disable 618
Manager.DefaultOptions.MaxRevsToGetInBulk = 10;
Manager.DefaultOptions.MaxOpenHttpConnections = 8;
Manager.DefaultOptions.RequestTimeout = TimeSpan.FromSeconds(5);
CreatePullAndTest((int)(Manager.DefaultOptions.MaxRevsToGetInBulk * 1.5), remoteDb, repl =>
{
WriteDebug("Document count increased to {0} with last sequence '{1}'", database.GetDocumentCount(), repl.LastSequence);
Assert.IsTrue(database.GetDocumentCount() > 0, "Didn't get docs from second bulk get batch");
Assert.AreEqual(gotSequence, Int32.Parse(repl.LastSequence), "LastSequence was advanced");
});
#pragma warning restore 618
Sleep(500);
fakeFactory.HttpHandler.ClearResponders();
var pull = database.CreatePullReplication(remoteDb.RemoteUri);
RunReplication(pull);
Assert.AreEqual(pull.ChangesCount, pull.CompletedChangesCount);
Assert.AreNotEqual(pull.LastSequence, "0");
}
}
public void TestBulkPullTransientExceptionRecovery() {
if (!Boolean.Parse((string)Runtime.Properties["replicationTestsEnabled"]))
{
Assert.Inconclusive("Replication tests disabled.");
return;
}
var initialRowCount = SyncGatewayRowCount();
var fakeFactory = new MockHttpClientFactory(false);
FlowControl flow = new FlowControl(new FlowItem[]
{
new FunctionRunner<HttpResponseMessage>(() => {
Thread.Sleep(7000);
return new RequestCorrectHttpMessage();
}) { ExecutionCount = 2 },
new FunctionRunner<HttpResponseMessage>(() => {
fakeFactory.HttpHandler.ClearResponders();
return new RequestCorrectHttpMessage();
}) { ExecutionCount = 1 }
});
fakeFactory.HttpHandler.SetResponder("_bulk_get", (request) =>
flow.ExecuteNext<HttpResponseMessage>());
manager.DefaultHttpClientFactory = fakeFactory;
ManagerOptions.Default.RequestTimeout = TimeSpan.FromSeconds(5);
CreatePullAndTest(20, (repl) => Assert.IsTrue(database.DocumentCount - initialRowCount == 20, "Didn't recover from the error"));
}