예제 #1
0
        public void TokenMap_SimpleStrategy_With_Hosts_Without_Tokens()
        {
            var hosts = new List <Host>
            {
                { TestHelper.CreateHost("192.168.0.0", "dc1", "rack", new HashSet <string> {
                        "0"
                    }) },
                { TestHelper.CreateHost("192.168.0.1", "dc1", "rack", new string[0]) },
                { TestHelper.CreateHost("192.168.0.2", "dc1", "rack", new HashSet <string> {
                        "20"
                    }) }
            };
            var keyspaces = new List <KeyspaceMetadata>
            {
                TokenTests.CreateSimpleKeyspace("ks1", 10),
                TokenTests.CreateSimpleKeyspace("ks2", 2)
            };
            var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces);

            //the primary replica and the next
            var replicas = tokenMap.GetReplicas("ks1", new M3PToken(0));

            //The node without tokens should not be considered
            CollectionAssert.AreEqual(new byte[] { 0, 2 }, replicas.Select(TestHelper.GetLastAddressByte));
            replicas = tokenMap.GetReplicas("ks1", new M3PToken(-100));
            CollectionAssert.AreEqual(new byte[] { 0, 2 }, replicas.Select(TestHelper.GetLastAddressByte));
            //Greater than the greatest token
            replicas = tokenMap.GetReplicas("ks1", new M3PToken(500000));
            CollectionAssert.AreEqual(new byte[] { 0, 2 }, replicas.Select(TestHelper.GetLastAddressByte));

            //The next replica should be the first
            replicas = tokenMap.GetReplicas("ks1", new M3PToken(20));
            CollectionAssert.AreEqual(new byte[] { 2, 0 }, replicas.Select(TestHelper.GetLastAddressByte));
        }
예제 #2
0
        public void TokenMap_SimpleStrategy_With_Keyspace_Test()
        {
            var hosts = new List <Host>
            {
                { TestHelper.CreateHost("192.168.0.0", "dc1", "rack", new HashSet <string> {
                        "0"
                    }) },
                { TestHelper.CreateHost("192.168.0.1", "dc1", "rack", new HashSet <string> {
                        "10"
                    }) },
                { TestHelper.CreateHost("192.168.0.2", "dc1", "rack", new HashSet <string> {
                        "20"
                    }) }
            };
            var keyspaces = new List <KeyspaceMetadata>
            {
                TokenTests.CreateSimpleKeyspace("ks1", 2),
                TokenTests.CreateSimpleKeyspace("ks2", 10)
            };
            var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces);

            //the primary replica and the next
            var replicas = tokenMap.GetReplicas("ks1", new M3PToken(0));

            Assert.AreEqual("0,1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
            replicas = tokenMap.GetReplicas("ks1", new M3PToken(-100));
            Assert.AreEqual("0,1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
            //Greater than the greatest token
            replicas = tokenMap.GetReplicas("ks1", new M3PToken(500000));
            Assert.AreEqual("0,1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));

            //The next replica should be the first
            replicas = tokenMap.GetReplicas("ks1", new M3PToken(20));
            Assert.AreEqual("2,0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));

            //The closest replica and the next
            replicas = tokenMap.GetReplicas("ks1", new M3PToken(19));
            Assert.AreEqual("2,0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));

            //Even if the replication factor is greater than the ring, it should return only ring size
            replicas = tokenMap.GetReplicas("ks2", new M3PToken(5));
            Assert.AreEqual("1,2,0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));

            //The primary replica only as the keyspace was not found
            replicas = tokenMap.GetReplicas(null, new M3PToken(0));
            Assert.AreEqual("0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
            replicas = tokenMap.GetReplicas(null, new M3PToken(10));
            Assert.AreEqual("1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
            replicas = tokenMap.GetReplicas("ks_does_not_exist", new M3PToken(20));
            Assert.AreEqual("2", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
            replicas = tokenMap.GetReplicas(null, new M3PToken(19));
            Assert.AreEqual("2", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
        }
예제 #3
0
        public void TokenMap_Build_SimpleStrategy_Adjacent_Ranges_Test()
        {
            var hosts = new[]
            {
                //0 and 100 are adjacent
                TestHelper.CreateHost("192.168.0.1", "dc1", "rack1", new HashSet <string> {
                    "0", "100", "1000"
                }),
                TestHelper.CreateHost("192.168.0.2", "dc1", "rack1", new HashSet <string> {
                    "200", "2000", "20000"
                }),
                TestHelper.CreateHost("192.168.0.3", "dc1", "rack1", new HashSet <string> {
                    "300", "3000", "30000"
                })
            };
            var ks       = TokenTests.CreateSimpleKeyspace("ks1", 2);
            var map      = TokenMap.Build("Murmur3Partitioner", hosts, new[] { ks });
            var replicas = map.GetReplicas("ks1", new M3PToken(0));

            Assert.AreEqual(2, replicas.Count);
            //It should contain the first host and the second, even though the first host contains adjacent
            CollectionAssert.AreEqual(new byte[] { 1, 2 }, replicas.Select(TestHelper.GetLastAddressByte));
        }
예제 #4
0
        public void RefreshSingleKeyspace_Should_BuildTokenMap_When_TokenMapIsNull()
        {
            var keyspaces = new ConcurrentDictionary <string, KeyspaceMetadata>();

            keyspaces.GetOrAdd("ks1", TokenTests.CreateSimpleKeyspace("ks1", 1));
            var schemaParser = new FakeSchemaParser(keyspaces);
            var metadata     = new Metadata(new Configuration(), schemaParser)
            {
                Partitioner = "Murmur3Partitioner"
            };

            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.1"), 9042));;
            metadata.Hosts.First().SetInfo(new TestHelper.DictionaryBasedRow(new Dictionary <string, object>
            {
                { "data_center", "dc1" },
                { "rack", "rack1" },
                { "tokens", GenerateTokens(1, 256) },
                { "release_version", "3.11.1" }
            }));

            Assert.IsNull(metadata.TokenToReplicasMap);
            metadata.RefreshSingleKeyspace(true, "ks1").GetAwaiter().GetResult();
            Assert.NotNull(metadata.TokenToReplicasMap);
        }
예제 #5
0
        public void TokenMapUpdates_Should_FunctionCorrectly_When_MultipleThreadsCallingRebuildAndUpdateKeyspace()
        {
            var keyspaces = new ConcurrentDictionary <string, KeyspaceMetadata>();

            // unique configurations
            keyspaces.AddOrUpdate("ks1", TokenTests.CreateSimpleKeyspace("ks1", 2), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks2", TokenTests.CreateSimpleKeyspace("ks2", 10), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks3", TokenTests.CreateSimpleKeyspace("ks3", 5), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks4", TokenTests.CreateNetworkTopologyKeyspace("ks4", new Dictionary <string, int> {
                { "dc1", 2 }, { "dc2", 2 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks5", TokenTests.CreateNetworkTopologyKeyspace("ks5", new Dictionary <string, int> {
                { "dc1", 1 }, { "dc2", 2 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks6", TokenTests.CreateNetworkTopologyKeyspace("ks6", new Dictionary <string, int> {
                { "dc1", 1 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);

            // duplicate configurations
            keyspaces.AddOrUpdate("ks7", TokenTests.CreateNetworkTopologyKeyspace("ks7", new Dictionary <string, int> {
                { "dc1", 2 }, { "dc2", 2 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks8", TokenTests.CreateNetworkTopologyKeyspace("ks8", new Dictionary <string, int> {
                { "dc1", 1 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks9", TokenTests.CreateNetworkTopologyKeyspace("ks9", new Dictionary <string, int> {
                { "dc1", 1 }, { "dc2", 2 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks10", TokenTests.CreateSimpleKeyspace("ks10", 10), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks11", TokenTests.CreateSimpleKeyspace("ks11", 2), (s, keyspaceMetadata) => keyspaceMetadata);

            var schemaParser = new FakeSchemaParser(keyspaces);
            var metadata     = new Metadata(new Configuration(), schemaParser)
            {
                Partitioner = "Murmur3Partitioner"
            };

            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.1"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.2"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.3"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.4"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.5"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.6"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.7"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.8"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.9"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.10"), 9042));
            var initialToken = 1;

            foreach (var h in metadata.Hosts)
            {
                h.SetInfo(new TestHelper.DictionaryBasedRow(new Dictionary <string, object>
                {
                    { "data_center", initialToken % 2 == 0 ? "dc1" : "dc2" },
                    { "rack", "rack1" },
                    { "tokens", GenerateTokens(initialToken, 256) },
                    { "release_version", "3.11.1" }
                }));
                initialToken++;
            }
            metadata.RefreshKeyspaces().GetAwaiter().GetResult();
            var expectedTokenMap = metadata.TokenToReplicasMap;

            Assert.NotNull(expectedTokenMap);
            var bag   = new ConcurrentBag <string>();
            var tasks = new List <Task>();

            for (var i = 0; i < 100; i++)
            {
                var index = i;
                tasks.Add(Task.Factory.StartNew(
                              () =>
                {
                    for (var j = 0; j < 11; j++)
                    {
                        if (j % 5 == 0)
                        {
                            metadata.RefreshKeyspaces().GetAwaiter().GetResult();
                        }
                        else if (j % 2 == 1)
                        {
                            if (bag.TryTake(out var ksName))
                            {
                                if (keyspaces.TryRemove(ksName, out var ks))
                                {
                                    if (!metadata.RemoveKeyspace(ks.Name).GetAwaiter().GetResult())
                                    {
                                        // one rebuild finished between keyspace remove and metadata removekeyspace
                                    }
                                }
                            }
                        }
                        else
                        {
                            var keyspaceName = $"ks_____{index}_____{j}";
                            var ks           = TokenTests.CreateSimpleKeyspace(keyspaceName, (index * j) % 10);
                            keyspaces.AddOrUpdate(
                                keyspaceName,
                                ks,
                                (s, keyspaceMetadata) => ks);
                            ks = metadata.RefreshSingleKeyspace(true, keyspaceName).GetAwaiter().GetResult();
                            if (ks == null)
                            {
                                throw new Exception($"refresh for {keyspaceName} returned null.");
                            }
                            bag.Add(keyspaceName);
                        }
                    }
                },
                              TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach));
            }
            Task.WaitAll(tasks.ToArray());
            AssertSameReplicas(keyspaces.Values, expectedTokenMap, metadata.TokenToReplicasMap);
        }
예제 #6
0
        public void Build_Should_OnlyCallOncePerReplicationConfiguration_When_MultipleKeyspacesWithSameReplicationOptions()
        {
            var hosts = new List <Host>
            {
                { TestHelper.CreateHost("192.168.0.0", "dc1", "rack", new HashSet <string> {
                        "0"
                    }) },
                { TestHelper.CreateHost("192.168.0.1", "dc1", "rack", new HashSet <string> {
                        "10"
                    }) },
                { TestHelper.CreateHost("192.168.0.2", "dc1", "rack", new HashSet <string> {
                        "20"
                    }) },
                { TestHelper.CreateHost("192.168.0.3", "dc2", "rack", new HashSet <string> {
                        "30"
                    }) },
                { TestHelper.CreateHost("192.168.0.4", "dc2", "rack", new HashSet <string> {
                        "40"
                    }) }
            };

            var factory   = new ProxyReplicationStrategyFactory();
            var keyspaces = new List <KeyspaceMetadata>
            {
                // unique configurations
                TokenTests.CreateSimpleKeyspace("ks1", 2, factory),
                TokenTests.CreateSimpleKeyspace("ks2", 10, factory),
                TokenTests.CreateSimpleKeyspace("ks3", 5, factory),
                TokenTests.CreateNetworkTopologyKeyspace("ks4", new Dictionary <string, int> {
                    { "dc1", 2 }, { "dc2", 2 }
                }, factory),
                TokenTests.CreateNetworkTopologyKeyspace("ks5", new Dictionary <string, int> {
                    { "dc1", 1 }, { "dc2", 2 }
                }, factory),
                TokenTests.CreateNetworkTopologyKeyspace("ks6", new Dictionary <string, int> {
                    { "dc1", 1 }
                }, factory),

                // duplicate configurations
                TokenTests.CreateNetworkTopologyKeyspace("ks7", new Dictionary <string, int> {
                    { "dc1", 2 }, { "dc2", 2 }
                }, factory),
                TokenTests.CreateNetworkTopologyKeyspace("ks8", new Dictionary <string, int> {
                    { "dc1", 1 }
                }, factory),
                TokenTests.CreateNetworkTopologyKeyspace("ks9", new Dictionary <string, int> {
                    { "dc1", 1 }, { "dc2", 2 }
                }, factory),
                TokenTests.CreateSimpleKeyspace("ks10", 10, factory),
                TokenTests.CreateSimpleKeyspace("ks11", 2, factory)
            };
            var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces);

            var proxyStrategies = keyspaces.Select(k => (ProxyReplicationStrategy)k.Strategy).ToList();

            Assert.AreEqual(6, proxyStrategies.Count(strategy => strategy.Calls > 0));

            AssertOnlyOneStrategyIsCalled(proxyStrategies, 0, 10);
            AssertOnlyOneStrategyIsCalled(proxyStrategies, 1, 9);
            AssertOnlyOneStrategyIsCalled(proxyStrategies, 2);
            AssertOnlyOneStrategyIsCalled(proxyStrategies, 3, 6);
            AssertOnlyOneStrategyIsCalled(proxyStrategies, 4, 8);
            AssertOnlyOneStrategyIsCalled(proxyStrategies, 5, 7);
        }
예제 #7
0
        public void Should_UpdateKeyspacesAndTokenMapCorrectly_When_MultipleThreadsCallingRefreshKeyspace()
        {
            var keyspaces = new ConcurrentDictionary <string, KeyspaceMetadata>();

            // unique configurations
            keyspaces.AddOrUpdate("ks1", TokenTests.CreateSimpleKeyspace("ks1", 2), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks2", TokenTests.CreateSimpleKeyspace("ks2", 10), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks3", TokenTests.CreateSimpleKeyspace("ks3", 5), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks4", TokenTests.CreateNetworkTopologyKeyspace("ks4", new Dictionary <string, int> {
                { "dc1", 2 }, { "dc2", 2 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks5", TokenTests.CreateNetworkTopologyKeyspace("ks5", new Dictionary <string, int> {
                { "dc1", 1 }, { "dc2", 2 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks6", TokenTests.CreateNetworkTopologyKeyspace("ks6", new Dictionary <string, int> {
                { "dc1", 1 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);

            // duplicate configurations
            keyspaces.AddOrUpdate("ks7", TokenTests.CreateNetworkTopologyKeyspace("ks7", new Dictionary <string, int> {
                { "dc1", 2 }, { "dc2", 2 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks8", TokenTests.CreateNetworkTopologyKeyspace("ks8", new Dictionary <string, int> {
                { "dc1", 1 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks9", TokenTests.CreateNetworkTopologyKeyspace("ks9", new Dictionary <string, int> {
                { "dc1", 1 }, { "dc2", 2 }
            }), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks10", TokenTests.CreateSimpleKeyspace("ks10", 10), (s, keyspaceMetadata) => keyspaceMetadata);
            keyspaces.AddOrUpdate("ks11", TokenTests.CreateSimpleKeyspace("ks11", 2), (s, keyspaceMetadata) => keyspaceMetadata);

            var schemaParser = new FakeSchemaParser(keyspaces);
            var config       = new TestConfigurationBuilder
            {
                ConnectionFactory = new FakeConnectionFactory()
            }.Build();
            var metadata = new Metadata(config, schemaParser)
            {
                Partitioner = "Murmur3Partitioner"
            };

            metadata.ControlConnection = new ControlConnection(
                new ProtocolEventDebouncer(new TaskBasedTimerFactory(), TimeSpan.FromMilliseconds(20), TimeSpan.FromSeconds(100)),
                ProtocolVersion.V3,
                config,
                metadata);
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.1"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.2"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.3"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.4"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.5"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.6"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.7"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.8"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.9"), 9042));
            metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.10"), 9042));
            var initialToken = 1;

            foreach (var h in metadata.Hosts)
            {
                h.SetInfo(new TestHelper.DictionaryBasedRow(new Dictionary <string, object>
                {
                    { "data_center", initialToken % 2 == 0 ? "dc1" : "dc2" },
                    { "rack", "rack1" },
                    { "tokens", GenerateTokens(initialToken, 256) },
                    { "release_version", "3.11.1" }
                }));
                initialToken++;
            }
            metadata.RebuildTokenMapAsync(false, true).GetAwaiter().GetResult();
            var expectedTokenMap = metadata.TokenToReplicasMap;

            Assert.NotNull(expectedTokenMap);
            var bag   = new ConcurrentBag <string>();
            var tasks = new List <Task>();

            for (var i = 0; i < 100; i++)
            {
                var index = i;
                tasks.Add(Task.Factory.StartNew(
                              () =>
                {
                    for (var j = 0; j < 35; j++)
                    {
                        if (j % 10 == 0 && index % 2 == 0)
                        {
                            metadata.RefreshSchemaAsync().GetAwaiter().GetResult();
                        }
                        else if (j % 16 == 0)
                        {
                            if (bag.TryTake(out var ksName))
                            {
                                if (keyspaces.TryRemove(ksName, out var ks))
                                {
                                    metadata.RefreshSchemaAsync(ksName).GetAwaiter().GetResult();
                                    ks = metadata.GetKeyspace(ksName);
                                    if (ks != null)
                                    {
                                        throw new Exception($"refresh for {ks.Name} returned non null after refresh single.");
                                    }
                                }
                            }
                        }
                        else
                        if (j % 2 == 0)
                        {
                            if (bag.TryTake(out var ksName))
                            {
                                if (keyspaces.TryRemove(ksName, out var ks))
                                {
                                    metadata.ControlConnection.HandleKeyspaceRefreshLaterAsync(ks.Name).GetAwaiter().GetResult();
                                    ks = metadata.GetKeyspace(ksName);
                                    if (ks != null)
                                    {
                                        throw new Exception($"refresh for {ks.Name} returned non null after remove.");
                                    }
                                }
                            }
                        }
                        else
                        {
                            var keyspaceName = $"ks_____{index}_____{j}";
                            var ks           = TokenTests.CreateSimpleKeyspace(keyspaceName, (index * j) % 10);
                            keyspaces.AddOrUpdate(
                                keyspaceName,
                                ks,
                                (s, keyspaceMetadata) => ks);
                            metadata.ControlConnection.HandleKeyspaceRefreshLaterAsync(ks.Name).GetAwaiter().GetResult();
                            ks = metadata.GetKeyspace(ks.Name);
                            if (ks == null)
                            {
                                throw new Exception($"refresh for {keyspaceName} returned null after add.");
                            }
                            bag.Add(keyspaceName);
                        }
                    }
                },
                              TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach));
            }
            Task.WaitAll(tasks.ToArray());
            AssertSameReplicas(keyspaces.Values, expectedTokenMap, metadata.TokenToReplicasMap);
        }