public void TestBatcherBatchSize5()
{
doneSignal = new CountdownEvent(10);
inboxCapacity = 10;
processorDelay = TimeSpan.FromSeconds(1);
var scheduler = new SingleTaskThreadpoolScheduler();
var batcher = new Batcher <string>(new TaskFactory(scheduler),
inboxCapacity, processorDelay, TestBatcherBatchSize5Processor);
var objectsToQueue = new List <string>();
for (var i = 0; i < inboxCapacity * 10; i++)
{
objectsToQueue.Add(i.ToString());
if (objectsToQueue.Count == 5)
{
batcher.QueueObjects(objectsToQueue);
objectsToQueue.Clear();
}
}
var success = doneSignal.Wait(TimeSpan.FromSeconds(35));
Assert.IsTrue(success);
}
public void TestChangeTrackerGiveUp()
{
var factory = new MockHttpClientFactory();
var httpHandler = (MockHttpRequestHandler)factory.HttpHandler;
httpHandler.AddResponderThrowExceptionAllRequests();
var changeTrackerFinishedSignal = new CountdownEvent(1);
var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, null);
client.HttpClientFactory = factory;
var testUrl = GetReplicationURL();
var scheduler = new SingleTaskThreadpoolScheduler();
var remoteSession = new RemoteSession(new RemoteSessionContructorOptions {
BaseUrl = testUrl,
WorkExecutor = new TaskFactory(scheduler)
});
remoteSession.SetupHttpClientFactory(client.HttpClientFactory, database, "giveup");
remoteSession.Setup(new ReplicationOptions());
var changeTracker = ChangeTrackerFactory.Create(new ChangeTrackerOptions {
DatabaseUri = testUrl,
Mode = ChangeTrackerMode.OneShot,
IncludeConflicts = true,
Client = client,
RetryStrategy = new ExponentialBackoffStrategy(2),
WorkExecutor = new TaskFactory(scheduler),
RemoteSession = remoteSession
});
changeTracker.Start();
Assert.IsTrue(changeTrackerFinishedSignal.Wait(TimeSpan.FromSeconds(30)));
}
public void TestBatcherBatchSize5()
{
doneSignal = new CountdownEvent(10);
inboxCapacity = 10;
processorDelay = TimeSpan.FromSeconds(1);
var scheduler = new SingleTaskThreadpoolScheduler();
var batcher = new Batcher <string>(new BatcherOptions <string> {
WorkExecutor = new TaskFactory(scheduler),
Capacity = inboxCapacity,
Delay = processorDelay,
Processor = TestBatcherBatchSize5Processor
});
var objectsToQueue = new List <string>();
for (var i = 0; i < inboxCapacity * 10; i++)
{
objectsToQueue.Add(i.ToString());
if (objectsToQueue.Count == 5)
{
batcher.QueueObjects(objectsToQueue);
objectsToQueue.Clear();
}
}
var success = doneSignal.Wait(TimeSpan.FromSeconds(35));
success.Should().BeTrue("beacuse otherwise the batcher didn't run the correct number of times");
}
private void TestChangeTrackerBackoff(MockHttpClientFactory httpClientFactory)
{
var changeTrackerFinishedSignal = new CountdownEvent(1);
var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, null);
client.HttpClientFactory = httpClientFactory;
var testUrl = GetReplicationURL();
var scheduler = new SingleTaskThreadpoolScheduler();
var changeTracker = ChangeTrackerFactory.Create(new ChangeTrackerOptions {
DatabaseUri = testUrl,
Mode = ChangeTrackerMode.LongPoll,
IncludeConflicts = true,
Client = client,
RetryStrategy = new ExponentialBackoffStrategy(2),
WorkExecutor = new TaskFactory(scheduler)
});
changeTracker.Continuous = true;
changeTracker.Start();
// sleep for a few seconds
Sleep(8 * 1000);
// make sure we got less than 10 requests in those 8 seconds (if it was hammering, we'd get a lot more)
var handler = client.HttpRequestHandler;
Assert.Less(handler.CapturedRequests.Count, 10);
Assert.Greater(changeTracker.Backoff.NumAttempts, 0, String.Format("Observed attempts: {0}", changeTracker.Backoff.NumAttempts));
handler.ClearResponders();
handler.AddResponderReturnEmptyChangesFeed();
// at this point, the change tracker backoff should cause it to sleep for about 3 seconds
// and so lets wait 3 seconds until it wakes up and starts getting valid responses
Sleep(10 * 1000);
// now find the delta in requests received in a 2s period
int before = handler.CapturedRequests.Count;
Sleep(2 * 1000);
int after = handler.CapturedRequests.Count;
// assert that the delta is high, because at this point the change tracker should
// be hammering away
Assert.Greater((after - before), 25);
// the backoff numAttempts should have been reset to 0
Assert.IsTrue(changeTracker.Backoff.NumAttempts == 0);
changeTracker.Stop();
var success = changeTrackerFinishedSignal.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
}
private void RunChangeTrackerTransientError(
ChangeTrackerMode mode,
Int32 errorCode,
string statusMessage,
Int32 numExpectedChangeCallbacks)
{
var changeTrackerFinishedSignal = new CountdownEvent(1);
var changeReceivedSignal = new CountdownEvent(numExpectedChangeCallbacks);
var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, changeReceivedSignal);
MockHttpRequestHandler.HttpResponseDelegate sentinal = RunChangeTrackerTransientErrorDefaultResponder();
var responders = new List <MockHttpRequestHandler.HttpResponseDelegate>();
responders.Add(RunChangeTrackerTransientErrorDefaultResponder());
responders.Add(MockHttpRequestHandler.TransientErrorResponder(errorCode, statusMessage));
MockHttpRequestHandler.HttpResponseDelegate chainResponder = (request) =>
{
if (responders.Count > 0)
{
var responder = responders[0];
responders.RemoveAt(0);
return(responder(request));
}
return(sentinal(request));
};
var handler = client.HttpRequestHandler;
handler.SetResponder("_changes", chainResponder);
var testUrl = GetReplicationURL();
var scheduler = new SingleTaskThreadpoolScheduler();
var changeTracker = ChangeTrackerFactory.Create(new ChangeTrackerOptions {
DatabaseUri = testUrl,
Mode = mode,
IncludeConflicts = false,
Client = client,
RetryStrategy = new ExponentialBackoffStrategy(2),
WorkExecutor = new TaskFactory(scheduler)
});
changeTracker.Start();
var success = changeReceivedSignal.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
changeTracker.Stop();
success = changeTrackerFinishedSignal.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
}
private void ChangeTrackerTestWithMode(ChangeTrackerMode mode)
{
var changeTrackerFinishedSignal = new CountdownEvent(1);
var changeReceivedSignal = new CountdownEvent(1);
var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, changeReceivedSignal);
client.ReceivedChangeDelegate = (IDictionary <string, object> change) =>
{
Assert.IsTrue(change.ContainsKey("seq"));
Assert.AreEqual("1", change["seq"]);
};
var handler = client.HttpRequestHandler;
handler.SetResponder("_changes", (request) =>
{
var json = "{\"results\":[\n" +
"{\"seq\":\"1\",\"id\":\"doc1-138\",\"changes\":[{\"rev\":\"1-82d\"}]}],\n" +
"\"last_seq\":\"*:50\"}";
return(MockHttpRequestHandler.GenerateHttpResponseMessage(System.Net.HttpStatusCode.OK, null, json));
});
var testUrl = GetReplicationURL();
var scheduler = new SingleTaskThreadpoolScheduler();
var remoteSession = new RemoteSession(new RemoteSessionContructorOptions {
WorkExecutor = new TaskFactory(scheduler),
BaseUrl = testUrl
});
remoteSession.SetupHttpClientFactory(client.HttpClientFactory, database, "testwithmode");
remoteSession.Setup(new ReplicationOptions());
var changeTracker = ChangeTrackerFactory.Create(new ChangeTrackerOptions {
DatabaseUri = testUrl,
Mode = mode,
IncludeConflicts = false,
Client = client,
RetryStrategy = new ExponentialBackoffStrategy(2),
WorkExecutor = new TaskFactory(scheduler),
RemoteSession = remoteSession
});
changeTracker.ActiveOnly = true;
changeTracker.Start();
var success = changeReceivedSignal.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
changeTracker.Stop();
success = changeTrackerFinishedSignal.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
}
public void TestBatcherCancel()
{
var mre = new ManualResetEventSlim();
var scheduler = new SingleTaskThreadpoolScheduler();
var batcher = new Batcher <int>(new TaskFactory(scheduler), 5, 500, (inbox) =>
{
mre.Set();
});
batcher.QueueObject(0);
batcher.Clear();
Assert.False(mre.Wait(TimeSpan.FromSeconds(1)), "Batcher ran after being cancelled");
}
private void RunChangeTrackerTransientError(
ChangeTrackerMode mode,
Int32 errorCode,
string statusMessage,
Int32 numExpectedChangeCallbacks)
{
var changeTrackerFinishedSignal = new CountDownLatch(1);
var changeReceivedSignal = new CountDownLatch(numExpectedChangeCallbacks);
var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, changeReceivedSignal);
MockHttpRequestHandler.HttpResponseDelegate sentinal = RunChangeTrackerTransientErrorDefaultResponder();
var responders = new List <MockHttpRequestHandler.HttpResponseDelegate>();
responders.Add(RunChangeTrackerTransientErrorDefaultResponder());
responders.Add(MockHttpRequestHandler.TransientErrorResponder(errorCode, statusMessage));
MockHttpRequestHandler.HttpResponseDelegate chainResponder = (request) =>
{
if (responders.Count > 0)
{
var responder = responders[0];
responders.RemoveAt(0);
return(responder(request));
}
return(sentinal(request));
};
var handler = client.HttpRequestHandler;
handler.SetResponder("_changes", chainResponder);
var testUrl = GetReplicationURL();
var scheduler = new SingleTaskThreadpoolScheduler();
var changeTracker = new ChangeTracker(testUrl, mode, 0, false, client, new TaskFactory(scheduler));
changeTracker.UsePost = IsSyncGateway(testUrl);
changeTracker.Start();
var success = changeReceivedSignal.Await(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
changeTracker.Stop();
success = changeTrackerFinishedSignal.Await(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
}
private void TestChangeTrackerBackoff(MockHttpClientFactory httpClientFactory)
{
var changeTrackerFinishedSignal = new CountDownLatch(1);
var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, null);
client.HttpClientFactory = httpClientFactory;
var testUrl = GetReplicationURL();
var scheduler = new SingleTaskThreadpoolScheduler();
var changeTracker = new ChangeTracker(testUrl, ChangeTrackerMode.LongPoll, 0, false, client, new TaskFactory(scheduler));
changeTracker.UsePost = IsSyncGateway(testUrl);
changeTracker.Start();
// sleep for a few seconds
Sleep(10 * 1000);
// make sure we got less than 10 requests in those 10 seconds (if it was hammering, we'd get a lot more)
var handler = client.HttpRequestHandler;
Assert.IsTrue(handler.CapturedRequests.Count < 25);
Assert.IsTrue(changeTracker.backoff.NumAttempts > 0, "Observed attempts: {0}".Fmt(changeTracker.backoff.NumAttempts));
handler.ClearResponders();
handler.AddResponderReturnEmptyChangesFeed();
// at this point, the change tracker backoff should cause it to sleep for about 3 seconds
// and so lets wait 3 seconds until it wakes up and starts getting valid responses
Sleep(3 * 1000);
// now find the delta in requests received in a 2s period
int before = handler.CapturedRequests.Count;
Sleep(2 * 1000);
int after = handler.CapturedRequests.Count;
// assert that the delta is high, because at this point the change tracker should
// be hammering away
Assert.IsTrue((after - before) > 25, "{0} <= 25", (after - before));
// the backoff numAttempts should have been reset to 0
Assert.IsTrue(changeTracker.backoff.NumAttempts == 0);
changeTracker.Stop();
var success = changeTrackerFinishedSignal.Await(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
}
public void TestBatcherAddAfterCancel()
{
var evt = new CountdownEvent(2);
var scheduler = new SingleTaskThreadpoolScheduler();
var batcher = new Batcher <int>(new TaskFactory(scheduler), 5, 500, (inbox) =>
{
evt.Signal();
});
batcher.QueueObject(0);
batcher.Clear();
batcher.QueueObject(0);
Assert.False(evt.Wait(TimeSpan.FromSeconds(1)), "Batcher ran too many times");
Assert.True(evt.CurrentCount == 1, "Batcher never ran");
}
public void TestBatcherCancel()
{
var mre = new ManualResetEventSlim();
var scheduler = new SingleTaskThreadpoolScheduler();
var batcher = new Batcher <int>(new BatcherOptions <int> {
WorkExecutor = new TaskFactory(scheduler),
Capacity = 5,
Delay = TimeSpan.FromMilliseconds(500),
Processor = (inbox) => mre.Set()
});
batcher.QueueObject(0);
mre.Wait(1000).Should().BeTrue("because otherwise the batcher didn't initially run");
mre.Reset();
batcher.QueueObject(0);
batcher.Clear();
mre.Wait(TimeSpan.FromSeconds(1)).Should().BeFalse("because otherwise the batcher ran after being cancelled");
}
public void TestBatcherAddAfterCancel()
{
var evt = new CountdownEvent(1);
var scheduler = new SingleTaskThreadpoolScheduler();
var batcher = new Batcher <int>(new BatcherOptions <int> {
WorkExecutor = new TaskFactory(scheduler),
Capacity = 5,
Delay = TimeSpan.FromMilliseconds(500),
Processor = (inbox) => evt.Signal()
});
batcher.QueueObject(0);
evt.Wait(1000).Should().BeTrue("because otherwise the batcher didn't initially run");
evt.Reset(2);
batcher.QueueObject(0);
batcher.Clear();
batcher.QueueObject(0);
evt.Wait(TimeSpan.FromSeconds(1.5)).Should().BeFalse("because otherwise the batcher ran too many times");
evt.CurrentCount.Should().Be(1, "because otherwise the batcher never ran");
}
private void ChangeTrackerTestWithMode(ChangeTrackerMode mode)
{
var changeTrackerFinishedSignal = new CountDownLatch(1);
var changeReceivedSignal = new CountDownLatch(1);
var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, changeReceivedSignal);
client.ReceivedChangeDelegate = (IDictionary <string, object> change) =>
{
Assert.IsTrue(change.ContainsKey("seq"));
Assert.AreEqual("1", change["seq"]);
};
var handler = client.HttpRequestHandler;
handler.SetResponder("_changes", (request) =>
{
var json = "{\"results\":[\n" +
"{\"seq\":\"1\",\"id\":\"doc1-138\",\"changes\":[{\"rev\":\"1-82d\"}]}],\n" +
"\"last_seq\":\"*:50\"}";
return(MockHttpRequestHandler.GenerateHttpResponseMessage(System.Net.HttpStatusCode.OK, null, json));
});
var testUrl = GetReplicationURL();
var scheduler = new SingleTaskThreadpoolScheduler();
var changeTracker = new ChangeTracker(testUrl, mode, 0, false, client, new TaskFactory(scheduler));
changeTracker.UsePost = IsSyncGateway(testUrl);
changeTracker.Start();
var success = changeReceivedSignal.Await(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
changeTracker.Stop();
success = changeTrackerFinishedSignal.Await(TimeSpan.FromSeconds(30));
Assert.IsTrue(success);
}
// This is used by the listener
internal Replication ReplicationWithProperties(IDictionary <string, object> properties)
{
// Extract the parameters from the JSON request body:
// http://wiki.apache.org/couchdb/Replication
bool push, createTarget;
var results = new Dictionary <string, object>()
{
{ "database", null },
{ "remote", null },
{ "headers", null },
{ "authorizer", null }
};
Status result = ParseReplicationProperties(properties, out push, out createTarget, results);
if (result.IsError)
{
throw Misc.CreateExceptionAndLog(Log.To.Listener, result.Code, TAG, "Failed to create replication");
}
object continuousObj = properties.Get("continuous");
bool continuous = false;
if (continuousObj is bool)
{
continuous = (bool)continuousObj;
}
var scheduler = new SingleTaskThreadpoolScheduler();
Replication rep = null;
if (push)
{
rep = new Pusher((Database)results["database"], (Uri)results["remote"], continuous, new TaskFactory(scheduler));
}
else
{
rep = new Puller((Database)results["database"], (Uri)results["remote"], continuous, new TaskFactory(scheduler));
}
rep.Filter = properties.Get("filter") as string;
rep.FilterParams = properties.Get("query_params").AsDictionary <string, object>();
rep.DocIds = properties.Get("doc_ids").AsList <string>();
rep.Headers = new Dictionary <string, string>();
foreach (var header in results.Get("headers").AsDictionary <string, string>())
{
if (header.Key.ToLowerInvariant() == "cookie")
{
var cookie = default(Cookie);
if (CookieParser.TryParse(header.Value, ((Uri)results["remote"]).GetLeftPart(UriPartial.Authority), out cookie))
{
rep.SetCookie(cookie.Name, cookie.Value, cookie.Path, cookie.Expires, cookie.Secure, cookie.HttpOnly);
}
else
{
Log.To.Listener.W(TAG, "Invalid cookie string received ({0}), ignoring...", header.Value);
}
}
else
{
rep.Headers.Add(header.Key, header.Value);
}
}
rep.Headers = results.Get("headers").AsDictionary <string, string>();
rep.Authenticator = results.Get("authorizer") as IAuthenticator;
if (push)
{
((Pusher)rep).CreateTarget = createTarget;
}
var db = (Database)results["database"];
// If this is a duplicate, reuse an existing replicator:
var activeReplicators = default(IList <Replication>);
var existing = default(Replication);
if (db.ActiveReplicators.AcquireTemp(out activeReplicators))
{
existing = activeReplicators.FirstOrDefault(x => x.LocalDatabase == rep.LocalDatabase &&
x.RemoteUrl == rep.RemoteUrl && x.IsPull == rep.IsPull &&
x.RemoteCheckpointDocID().Equals(rep.RemoteCheckpointDocID()));
}
return(existing ?? rep);
}
internal Replication ReplicationWithProperties(IDictionary <string, object> properties)
{
// Extract the parameters from the JSON request body:
// http://wiki.apache.org/couchdb/Replication
bool push, createTarget;
var results = new Dictionary <string, object>()
{
{ "database", null },
{ "remote", null },
{ "headers", null },
{ "authorizer", null }
};
Status result = ParseReplicationProperties(properties, out push, out createTarget, results);
if (result.IsError)
{
throw new CouchbaseLiteException(result.Code);
}
object continuousObj = properties.Get("continuous");
bool continuous = false;
if (continuousObj is bool)
{
continuous = (bool)continuousObj;
}
var scheduler = new SingleTaskThreadpoolScheduler();
Replication rep = null;
if (push)
{
rep = new Pusher((Database)results["database"], (Uri)results["remote"], continuous, new TaskFactory(scheduler));
}
else
{
rep = new Puller((Database)results["database"], (Uri)results["remote"], continuous, new TaskFactory(scheduler));
}
rep.Filter = properties.Get("filter") as string;
rep.FilterParams = properties.Get("query_params") as IDictionary <string, object>;
rep.DocIds = properties.Get("doc_ids") as IEnumerable <string>;
rep.RequestHeaders = results.Get("headers") as IDictionary <string, object>;
rep.Authenticator = results.Get("authorizer") as IAuthenticator;
if (push)
{
((Pusher)rep).CreateTarget = createTarget;
}
var db = (Database)results["database"];
// If this is a duplicate, reuse an existing replicator:
var existing = db.ActiveReplicators.FirstOrDefault(x => x.LocalDatabase == rep.LocalDatabase &&
x.RemoteUrl == rep.RemoteUrl && x.IsPull == rep.IsPull &&
x.RemoteCheckpointDocID().Equals(rep.RemoteCheckpointDocID()));
return(existing ?? rep);
}
