private void RunChangeTrackerTransientError(
            ChangeTrackerMode mode,
            Int32 errorCode,
            string statusMessage,
            Int32 numExpectedChangeCallbacks)
        {
            var changeTrackerFinishedSignal = new CountdownEvent(1);
            var changeReceivedSignal        = new CountdownEvent(numExpectedChangeCallbacks);
            var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, changeReceivedSignal);

            MockHttpRequestHandler.HttpResponseDelegate sentinal = RunChangeTrackerTransientErrorDefaultResponder();

            var responders = new List <MockHttpRequestHandler.HttpResponseDelegate>();

            responders.Add(RunChangeTrackerTransientErrorDefaultResponder());
            responders.Add(MockHttpRequestHandler.TransientErrorResponder(errorCode, statusMessage));

            MockHttpRequestHandler.HttpResponseDelegate chainResponder = (request) =>
            {
                if (responders.Count > 0)
                {
                    var responder = responders[0];
                    responders.RemoveAt(0);
                    return(responder(request));
                }

                return(sentinal(request));
            };

            var handler = client.HttpRequestHandler;

            handler.SetResponder("_changes", chainResponder);

            var testUrl       = GetReplicationURL();
            var scheduler     = new SingleTaskThreadpoolScheduler();
            var remoteSession = new RemoteSession(new RemoteSessionContructorOptions {
                WorkExecutor = new TaskFactory(scheduler),
                BaseUrl      = testUrl
            });

            remoteSession.SetupHttpClientFactory(client.HttpClientFactory, database);
            remoteSession.Setup(new ReplicationOptions());
            var changeTracker = ChangeTrackerFactory.Create(new ChangeTrackerOptions {
                DatabaseUri      = testUrl,
                Mode             = mode,
                IncludeConflicts = false,
                Client           = client,
                RetryStrategy    = new ExponentialBackoffStrategy(2),
                WorkExecutor     = new TaskFactory(scheduler),
                RemoteSession    = remoteSession
            });

            changeTracker.Start();

            var success = changeReceivedSignal.Wait(TimeSpan.FromSeconds(30));

            Assert.IsTrue(success);

            changeTracker.Stop(ErrorResolution.Ignore);

            success = changeTrackerFinishedSignal.Wait(TimeSpan.FromSeconds(30));
            Assert.IsTrue(success);
        }
        private void TestChangeTrackerBackoff(MockHttpClientFactory httpClientFactory)
        {
            var changeTrackerFinishedSignal = new CountdownEvent(1);
            var client = new ChangeTrackerTestClient(changeTrackerFinishedSignal, null);

            client.HttpClientFactory = httpClientFactory;

            var testUrl   = GetReplicationURL();
            var scheduler = new SingleTaskThreadpoolScheduler();
            var opts      = new RemoteSessionContructorOptions {
                BaseUrl      = testUrl,
                WorkExecutor = new TaskFactory(scheduler)
            };
            var remoteSession = new RemoteSession(opts);

            remoteSession.SetupHttpClientFactory(client.HttpClientFactory, database);
            remoteSession.Setup(new ReplicationOptions());
            var changeTracker = ChangeTrackerFactory.Create(new ChangeTrackerOptions {
                DatabaseUri      = testUrl,
                Mode             = ChangeTrackerMode.LongPoll,
                IncludeConflicts = true,
                Client           = client,
                RetryStrategy    = new ExponentialBackoffStrategy(2),
                WorkExecutor     = new TaskFactory(scheduler),
                RemoteSession    = remoteSession
            });

            changeTracker.Continuous = true;

            changeTracker.Start();

            // sleep for a few seconds
            Sleep(5 * 1000);

            // make sure we got less than 10 requests in those 8 seconds (if it was hammering, we'd get a lot more)
            var handler = client.HttpRequestHandler;

            Assert.Less(handler.CapturedRequests.Count, 10);

            handler.ClearResponders();
            handler.AddResponderReturnEmptyChangesFeed();
            handler.ResponseDelayMilliseconds = 100;

            // at this point, the change tracker backoff should cause it to sleep for about 3 seconds
            // and so lets wait 3 seconds until it wakes up and starts getting valid responses
            Sleep(10 * 1000);

            // now find the delta in requests received in a 2s period
            int before = handler.CapturedRequests.Count;

            Sleep(2 * 1000);
            int after = handler.CapturedRequests.Count;

            changeTracker.Stop(ErrorResolution.Ignore);

            // assert that the delta is high, because at this point the change tracker should
            // be hammering away
            Assert.Greater((after - before), 10);

            // the backoff numAttempts should have been reset to 0
            Assert.IsTrue(changeTracker.Backoff.NumAttempts == 0);

            var success = changeTrackerFinishedSignal.Wait(TimeSpan.FromSeconds(30));

            Assert.IsTrue(success);
        }