Esempio n. 1
0
        public void TestMultipleCommandsCarryOverMultipleBucketsAndThenAgeOut()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-F");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-F");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingConcurrency-F");

            stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(30).Subscribe(GetSubscriber(output, latch));

            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
            Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
            Command cmd4 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);

            cmd1.Observe();
            Time.Wait(100); // bucket roll
            cmd2.Observe();
            Time.Wait(100);
            cmd3.Observe();
            Time.Wait(100);
            cmd4.Observe();
            Assert.True(latch.Wait(10000));
            Assert.Equal(0, stream.LatestRollingMax);
        }
        public void TestFallbackMissing()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-K");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-K");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-K");

            stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));

            Command cmd = Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_MISSING);

            cmd.Observe();

            try
            {
                Assert.True(latch.Wait(10000));
            }
            catch (Exception)
            {
                Assert.True(false, "Interrupted ex");
            }

            output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());

            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
Esempio n. 3
0
        public void TestStartsAndEndsInSameBucketSemaphoreIsolated()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-C");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-C");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingConcurrency-C");

            stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));

            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10, ExecutionIsolationStrategy.SEMAPHORE);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 14, ExecutionIsolationStrategy.SEMAPHORE);

            cmd1.Observe();
            Time.Wait(1);
            cmd2.Observe();

            Assert.True(latch.Wait(10000));

            // since commands run in semaphore isolation, they are not tracked by threadpool metrics
            Assert.Equal(0, stream.LatestRollingMax);
        }
Esempio n. 4
0
        public void TestConcurrencyStreamProperlyFiltersOutSemaphoreRejections()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-I");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-I");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingConcurrency-I");

            stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));



            //10 commands executed concurrently on different caller threads should saturate semaphore
            //once these are in-flight, execute 10 more concurrently on new caller threads.
            //since these are semaphore-rejected, the max concurrency should be 10

            List <Command> saturators = new List <Command>();

            for (int i = 0; i < 10; i++)
            {
                saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 400, ExecutionIsolationStrategy.SEMAPHORE));
            }

            List <Command> rejected = new List <Command>();

            for (int i = 0; i < 10; i++)
            {
                rejected.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100, ExecutionIsolationStrategy.SEMAPHORE));
            }

            foreach (Command saturatingCmd in saturators)
            {
                Task t = new Task(() =>
                {
                    saturatingCmd.Observe();
                }, CancellationToken.None, TaskCreationOptions.LongRunning);
                t.Start();
            }


            Time.Wait(30);

            foreach (Command rejectedCmd in rejected)
            {
                Task.Run(() => rejectedCmd.Observe());
            }

            Assert.True(latch.Wait(10000));
            output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());

            foreach (Command rejectedCmd in rejected)
            {
                Assert.True(rejectedCmd.IsResponseSemaphoreRejected || rejectedCmd.IsResponseShortCircuited);
            }
            //should be 0 since all are executed in a semaphore
            Assert.Equal(0, stream.LatestRollingMax);
        }
        public void TestEmptyStreamProducesZeros()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-A");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-A");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-A");

            stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));

            // no writes
            try
            {
                Assert.True(latch.Wait(10000));
            }
            catch (Exception)
            {
                Assert.True(false, "Interrupted ex");
            }

            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
Esempio n. 6
0
        public void TestOneCommandCarriesOverToNextBucket()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-D");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-D");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingConcurrency-D");

            stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));

            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 560);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
            Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 75);

            cmd1.Observe();
            Time.Wait(150); // bucket roll
            cmd2.Observe();
            Time.Wait(1);
            cmd3.Observe();

            Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
            Assert.Equal(3, stream.LatestRollingMax);
        }
Esempio n. 7
0
        public void TestStartsAndEndsInSameBucketProduceValue()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-B");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-B");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingConcurrency-B");

            stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));



            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);

            cmd1.Observe();
            Time.Wait(1);
            cmd2.Observe();

            Assert.True(latch.Wait(10000));
            Assert.Equal(2, stream.LatestRollingMax);
        }
Esempio n. 8
0
        public void TestRequestFromCache()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-F");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-F");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-F");

            stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));

            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
            Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);

            cmd1.Observe();
            cmd2.Observe();
            cmd3.Observe();
            Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
            output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());

            // RESPONSE_FROM_CACHE should not show up at all in thread pool counters - just the success
            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
Esempio n. 9
0
        public async void TestMultipleEventsOverTimeGetStoredAndDoNotAgeOut()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-M");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-M");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-M");
            CountdownEvent          latch         = new CountdownEvent(1);
            var observer = new LatchedObserver(output, latch);

            stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 10);

            latchSubscription = stream.Observe().Take(20 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
            Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");

            await cmd1.Observe();

            await cmd2.Observe();

            Assert.True(latch.Wait(20000), "CountdownEvent was not set!");

            output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());

            // all commands should not have aged out
            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
Esempio n. 10
0
        public async void TestRequestFromCache()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-F");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-F");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-F");

            CountdownEvent latch    = new CountdownEvent(1);
            var            observer = new LatchedObserver(output, latch);

            stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);

            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
            Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);

            latchSubscription = stream.Observe().Subscribe(observer);
            Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");

            await cmd1.Observe();

            await cmd2.Observe();

            await cmd3.Observe();

            Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");

            // RESPONSE_FROM_CACHE should not show up at all in thread pool counters - just the success
            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
        public void TestMultipleEventsOverTimeGetStoredAndDoNotAgeOut()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-M");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-M");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-M");

            stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(30).Subscribe(new LatchedObserver(output, latch));

            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 10);

            cmd1.Observe();
            cmd2.Observe();
            try
            {
                Assert.True(latch.Wait(10000));
            }
            catch (Exception)
            {
                Assert.True(false, "Interrupted ex");
            }

            output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());

            // all commands should have aged out
            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
Esempio n. 12
0
        public void TestConcurrencyStreamProperlyFiltersOutResponseFromCache()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-G");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-G");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingConcurrency-G");

            stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));

            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
            Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
            Command cmd4 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);

            cmd1.Observe();
            Time.Wait(5);
            cmd2.Observe();
            cmd3.Observe();
            cmd4.Observe();

            Assert.True(latch.Wait(10000));
            output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
            Assert.True(cmd2.IsResponseFromCache);
            Assert.True(cmd3.IsResponseFromCache);
            Assert.True(cmd4.IsResponseFromCache);
            Assert.Equal(1, stream.LatestRollingMax);
        }
Esempio n. 13
0
        public void TestSemaphoreRejected()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-H");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-H");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-H");

            stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));

            //10 commands will saturate semaphore when called from different threads.
            //submit 2 more requests and they should be SEMAPHORE_REJECTED
            //should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes

            List <Command> saturators = new List <Command>();

            for (int i = 0; i < 10; i++)
            {
                saturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 300, ExecutionIsolationStrategy.SEMAPHORE));
            }

            CommandStreamTest.Command rejected1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
            CommandStreamTest.Command rejected2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);

            foreach (Command c in saturators)
            {
                Task t = new Task(() =>
                {
                    c.Observe();
                }, CancellationToken.None, TaskCreationOptions.LongRunning);
                t.Start();

                Task.Run(() => c.Observe());
            }

            Time.Wait(10);

            Task.Run(() => rejected1.Observe());
            Task.Run(() => rejected2.Observe());

            try
            {
                Assert.True(latch.Wait(10000));
            }
            catch (Exception)
            {
                Assert.True(false, "Interrupted ex");
            }

            output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
            Assert.True(rejected1.IsResponseSemaphoreRejected);
            Assert.True(rejected2.IsResponseSemaphoreRejected);

            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
        public void TestSingleBadRequest()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-E");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-E");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-E");

            stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));

            Command cmd = Command.From(groupKey, key, HystrixEventType.BAD_REQUEST);

            cmd.Observe();

            try
            {
                Assert.True(latch.Wait(10000));
            }
            catch (Exception)
            {
                Assert.True(false, "Interrupted ex");
            }

            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
        public void TestMultipleCommandsCarryOverMultipleBucketsAndThenAgeOut()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-F");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-F");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingConcurrency-F");
            CountdownEvent          latch         = new CountdownEvent(1);
            var observer = new LatchedObserver(output, latch);

            stream            = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
            latchSubscription = stream.Observe().Take(20 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
            Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");

            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
            Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
            Command cmd4 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);

            Task t1 = cmd1.ExecuteAsync();

            WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
            Task t2 = cmd2.ExecuteAsync();

            WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
            Task t3 = cmd3.ExecuteAsync();

            WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
            Task t4 = cmd4.ExecuteAsync();

            Task.WaitAll(t1, t2, t3, t4);

            Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
            Assert.Equal(0, stream.LatestRollingMax);
        }
        public void TestConcurrencyStreamProperlyFiltersOutResponseFromCache()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-G");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-G");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingConcurrency-G");
            CountdownEvent          latch         = new CountdownEvent(1);
            var observer = new LatchedObserver(output, latch);

            stream            = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
            latchSubscription = stream.Observe().Subscribe(observer);
            Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");

            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
            Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
            Command cmd4 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);

            cmd1.Execute();
            cmd2.Execute();
            cmd3.Execute();
            cmd4.Execute();

            Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
            Assert.True(cmd2.IsResponseFromCache);
            Assert.True(cmd3.IsResponseFromCache);
            Assert.True(cmd4.IsResponseFromCache);
            Assert.Equal(1, stream.LatestRollingMax);
        }
Esempio n. 17
0
        public void TestMultipleEventsOverTimeGetStoredAndAgeOut()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-M");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-M");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingCounter-M");

            stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 250);
            stream.StartCachingStreamValuesIfUnstarted();

            // by doing a take(20), we ensure that all rolling counts go back to 0
            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(20).Subscribe(GetSubscriber(output, latch));

            CommandStreamTest.Command cmd1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
            CommandStreamTest.Command cmd2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 10);

            cmd1.Observe();
            cmd2.Observe();
            try
            {
                Assert.True(latch.Wait(10000));
            }
            catch (Exception)
            {
                Assert.False(true, "Interrupted ex");
            }

            // all commands should have aged out
            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
        public void TestOneCommandCarriesOverToNextBucket()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-D");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-D");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingConcurrency-D");
            CountdownEvent          latch         = new CountdownEvent(1);
            var observer = new LatchedObserver(output, latch);

            stream            = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
            latchSubscription = stream.Observe().Subscribe(observer);
            Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");

            Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 560);
            Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
            Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 75);

            Task t1 = cmd1.ExecuteAsync();

            // Time.Wait(150); // bucket roll
            Assert.True(WaitForObservableToUpdate(stream.Observe(), 1, 500, output), "Stream update took to long");
            Task t2 = cmd2.ExecuteAsync();
            Task t3 = cmd3.ExecuteAsync();

            Task.WaitAll(t1, t2, t3);
            Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
            Assert.Equal(3, stream.LatestRollingMax);
        }
Esempio n. 19
0
        public void TestFallbackMissing()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-K");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-K");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingCounter-K");

            stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));

            CommandStreamTest.Command cmd = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_MISSING);

            cmd.Observe();
            try
            {
                Assert.True(latch.Wait(10000));
            }
            catch (Exception)
            {
                Assert.False(true, "Interrupted ex");
            }

            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
Esempio n. 20
0
        public void TestThreadPoolRejected()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-I");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-I");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingCounter-I");

            stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));

            // 10 commands will saturate threadpools when called concurrently.
            // submit 2 more requests and they should be THREADPOOL_REJECTED
            // should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes
            List <CommandStreamTest.Command> saturators = new List <CommandStreamTest.Command>();

            for (int i = 0; i < 10; i++)
            {
                saturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 200));
            }

            CommandStreamTest.Command rejected1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
            CommandStreamTest.Command rejected2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);

            foreach (CommandStreamTest.Command saturator in saturators)
            {
                saturator.Observe();
            }

            try
            {
                Time.Wait(100);
            }
            catch (Exception ie)
            {
                Assert.False(true, ie.Message);
            }

            rejected1.Observe();
            rejected2.Observe();
            try
            {
                Assert.True(latch.Wait(10000));
            }
            catch (Exception)
            {
                Assert.False(true, "Interrupted ex");
            }

            output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
            Assert.True(rejected1.IsResponseThreadPoolRejected);
            Assert.True(rejected2.IsResponseThreadPoolRejected);

            // none of these got executed on a thread-pool, so thread pool metrics should be 0
            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(10, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
            public static Command From(IHystrixCommandGroupKey groupKey, IHystrixCommandKey key, HystrixEventType desiredEventType, int latency,
                                       ExecutionIsolationStrategy isolationStrategy,
                                       HystrixEventType desiredFallbackEventType, int fallbackLatency)
            {
                HystrixThreadPoolOptions topts = new HystrixThreadPoolOptions()
                {
                    CoreSize      = 10,
                    MaxQueueSize  = -1,
                    ThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(groupKey.Name)
                };

                HystrixCommandOptions setter = new HystrixCommandOptions()
                {
                    GroupKey   = groupKey,
                    CommandKey = key,

                    ExecutionTimeoutInMilliseconds                = 600,
                    ExecutionIsolationStrategy                    = isolationStrategy,
                    CircuitBreakerEnabled                         = true,
                    CircuitBreakerRequestVolumeThreshold          = 3,
                    MetricsHealthSnapshotIntervalInMilliseconds   = 100,
                    MetricsRollingStatisticalWindowInMilliseconds = 1000,
                    MetricsRollingStatisticalWindowBuckets        = 10,
                    RequestCacheEnabled = true,
                    RequestLogEnabled   = true,
                    FallbackIsolationSemaphoreMaxConcurrentRequests = 5,
                    ThreadPoolKey     = HystrixThreadPoolKeyDefault.AsKey(groupKey.Name),
                    ThreadPoolOptions = topts
                };



                String uniqueArg;

                switch (desiredEventType)
                {
                case HystrixEventType.SUCCESS:
                    uniqueArg = uniqueId.IncrementAndGet() + "";
                    return(new Command(setter, HystrixEventType.SUCCESS, latency, uniqueArg, desiredFallbackEventType, 0));

                case HystrixEventType.FAILURE:
                    uniqueArg = uniqueId.IncrementAndGet() + "";
                    return(new Command(setter, HystrixEventType.FAILURE, latency, uniqueArg, desiredFallbackEventType, fallbackLatency));

                case HystrixEventType.TIMEOUT:
                    uniqueArg = uniqueId.IncrementAndGet() + "";
                    return(new Command(setter, HystrixEventType.SUCCESS, 1000, uniqueArg, desiredFallbackEventType, fallbackLatency));

                case HystrixEventType.BAD_REQUEST:
                    uniqueArg = uniqueId.IncrementAndGet() + "";
                    return(new Command(setter, HystrixEventType.BAD_REQUEST, latency, uniqueArg, desiredFallbackEventType, 0));

                case HystrixEventType.RESPONSE_FROM_CACHE:
                    String arg = uniqueId.Value + "";
                    return(new Command(setter, HystrixEventType.SUCCESS, 0, arg, desiredFallbackEventType, 0));

                default:
                    throw new Exception("not supported yet");
                }
            }
Esempio n. 22
0
        public void TestEmptyStreamProducesZeros()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-A");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-A");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingConcurrency-A");

            stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));

            // no writes
            try
            {
                Assert.True(latch.Wait(10000));
            }
            catch (Exception)
            {
                Assert.False(true, "Interrupted ex");
            }

            Assert.Equal(0, stream.LatestRollingMax);
        }
Esempio n. 23
0
        public void TestSemaphoreRejected()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-H");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-H");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("RollingCounter-H");

            stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));

            // 10 commands will saturate semaphore when called from different threads.
            // submit 2 more requests and they should be SEMAPHORE_REJECTED
            // should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes
            List <Command> saturators = new List <Command>();

            for (int i = 0; i < 10; i++)
            {
                saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE));
            }

            Command rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
            Command rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);

            foreach (Command saturator in saturators)
            {
                Task t = new Task(
                    () =>
                {
                    saturator.Observe();
                }, CancellationToken.None,
                    TaskCreationOptions.LongRunning);
                t.Start();
            }

            try
            {
                Time.Wait(100);
            }
            catch (Exception ie)
            {
                Assert.False(true, ie.Message);
            }

            rejected1.Observe();
            rejected2.Observe();

            Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
            output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
            Assert.True(rejected1.IsResponseSemaphoreRejected);
            Assert.True(rejected2.IsResponseSemaphoreRejected);

            // none of these got executed on a thread-pool, so thread pool metrics should be 0
            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
Esempio n. 24
0
 public MultiPriceServiceCommand(IHystrixCommandGroupKey groupKey,
                                 ICollection <ICollapsedRequest <Price, int> > requests, IPriceService service,
                                 ILogger <MultiPriceServiceCommand> logger) : base(groupKey)
 {
     _logger   = logger;
     _service  = service;
     _requests = requests;
 }
Esempio n. 25
0
 private static HystrixCommandConfiguration SampleCommandConfiguration(
     IHystrixCommandKey commandKey,
     IHystrixThreadPoolKey threadPoolKey,
     IHystrixCommandGroupKey groupKey,
     IHystrixCommandOptions commandProperties)
 {
     return(HystrixCommandConfiguration.Sample(commandKey, threadPoolKey, groupKey, commandProperties));
 }
        public void TestFallbackRejection()
        {
            IHystrixCommandGroupKey groupKey      = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-L");
            IHystrixThreadPoolKey   threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-L");
            IHystrixCommandKey      key           = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-L");

            stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
            stream.StartCachingStreamValuesIfUnstarted();

            CountdownEvent latch = new CountdownEvent(1);

            stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));

            // fallback semaphore size is 5.  So let 5 commands saturate that semaphore, then
            // let 2 more commands go to fallback.  they should get rejected by the fallback-semaphore
            List <CommandStreamTest.Command> fallbackSaturators = new List <CommandStreamTest.Command>();

            for (int i = 0; i < 5; i++)
            {
                fallbackSaturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 400));
            }

            Command rejection1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0);
            Command rejection2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0);

            foreach (Command saturator in fallbackSaturators)
            {
                saturator.Observe();
            }

            try
            {
                Time.Wait(70);
            }
            catch (Exception ie)
            {
                Assert.True(false, ie.Message);
            }

            rejection1.Observe();
            rejection2.Observe();

            try
            {
                Assert.True(latch.Wait(10000));
            }
            catch (Exception)
            {
                Assert.True(false, "Interrupted ex");
            }

            output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());

            // all 7 commands executed on-thread, so should be executed according to thread-pool metrics
            Assert.Equal(2, stream.Latest.Length);
            Assert.Equal(7, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
            Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
        }
Esempio n. 27
0
 public MultiFortuneServiceCommand(IHystrixCommandGroupKey groupKey,
                                   ICollection <ICollapsedRequest <Fortune, int> > requests,
                                   IFortuneService fortuneService,
                                   ILogger <MultiFortuneServiceCommand> logger) : base(groupKey)
 {
     _fortuneService = fortuneService;
     _logger         = logger;
     _requests       = requests;
 }
Esempio n. 28
0
 public static Command From(
     IHystrixCommandGroupKey groupKey,
     IHystrixCommandKey key,
     HystrixEventType desiredEventType,
     int latency,
     ExecutionIsolationStrategy isolationStrategy)
 {
     return(From(groupKey, key, desiredEventType, latency, isolationStrategy, HystrixEventType.FALLBACK_SUCCESS, 0));
 }
Esempio n. 29
0
 public static Command From(
     IHystrixCommandGroupKey groupKey,
     IHystrixCommandKey key,
     HystrixEventType desiredEventType,
     int latency,
     HystrixEventType desiredFallbackEventType)
 {
     return(From(groupKey, key, desiredEventType, latency, ExecutionIsolationStrategy.THREAD, desiredFallbackEventType));
 }
Esempio n. 30
0
 public HystrixCommandOptions(
     IHystrixCommandGroupKey groupKey,
     IHystrixCommandKey key,
     IHystrixCommandOptions defaults = null,
     IHystrixDynamicOptions dynamic  = null)
     : this(key, defaults, dynamic)
 {
     GroupKey = groupKey;
 }