public void TokenMap_SimpleStrategy_With_Hosts_Without_Tokens() { var hosts = new List <Host> { { TestHelper.CreateHost("192.168.0.0", "dc1", "rack", new HashSet <string> { "0" }) }, { TestHelper.CreateHost("192.168.0.1", "dc1", "rack", new string[0]) }, { TestHelper.CreateHost("192.168.0.2", "dc1", "rack", new HashSet <string> { "20" }) } }; var keyspaces = new List <KeyspaceMetadata> { FakeSchemaParserFactory.CreateSimpleKeyspace("ks1", 10), FakeSchemaParserFactory.CreateSimpleKeyspace("ks2", 2) }; var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces); //the primary replica and the next var replicas = tokenMap.GetReplicas("ks1", new M3PToken(0)); //The node without tokens should not be considered CollectionAssert.AreEqual(new byte[] { 0, 2 }, replicas.Select(TestHelper.GetLastAddressByte)); replicas = tokenMap.GetReplicas("ks1", new M3PToken(-100)); CollectionAssert.AreEqual(new byte[] { 0, 2 }, replicas.Select(TestHelper.GetLastAddressByte)); //Greater than the greatest token replicas = tokenMap.GetReplicas("ks1", new M3PToken(500000)); CollectionAssert.AreEqual(new byte[] { 0, 2 }, replicas.Select(TestHelper.GetLastAddressByte)); //The next replica should be the first replicas = tokenMap.GetReplicas("ks1", new M3PToken(20)); CollectionAssert.AreEqual(new byte[] { 2, 0 }, replicas.Select(TestHelper.GetLastAddressByte)); }
public void TokenMap_SimpleStrategy_With_Keyspace_Test() { var hosts = new List <Host> { { TestHelper.CreateHost("192.168.0.0", "dc1", "rack", new HashSet <string> { "0" }) }, { TestHelper.CreateHost("192.168.0.1", "dc1", "rack", new HashSet <string> { "10" }) }, { TestHelper.CreateHost("192.168.0.2", "dc1", "rack", new HashSet <string> { "20" }) } }; var keyspaces = new List <KeyspaceMetadata> { FakeSchemaParserFactory.CreateSimpleKeyspace("ks1", 2), FakeSchemaParserFactory.CreateSimpleKeyspace("ks2", 10) }; var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces); //the primary replica and the next var replicas = tokenMap.GetReplicas("ks1", new M3PToken(0)); Assert.AreEqual("0,1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte))); replicas = tokenMap.GetReplicas("ks1", new M3PToken(-100)); Assert.AreEqual("0,1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte))); //Greater than the greatest token replicas = tokenMap.GetReplicas("ks1", new M3PToken(500000)); Assert.AreEqual("0,1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte))); //The next replica should be the first replicas = tokenMap.GetReplicas("ks1", new M3PToken(20)); Assert.AreEqual("2,0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte))); //The closest replica and the next replicas = tokenMap.GetReplicas("ks1", new M3PToken(19)); Assert.AreEqual("2,0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte))); //Even if the replication factor is greater than the ring, it should return only ring size replicas = tokenMap.GetReplicas("ks2", new M3PToken(5)); Assert.AreEqual("1,2,0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte))); //The primary replica only as the keyspace was not found replicas = tokenMap.GetReplicas(null, new M3PToken(0)); Assert.AreEqual("0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte))); replicas = tokenMap.GetReplicas(null, new M3PToken(10)); Assert.AreEqual("1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte))); replicas = tokenMap.GetReplicas("ks_does_not_exist", new M3PToken(20)); Assert.AreEqual("2", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte))); replicas = tokenMap.GetReplicas(null, new M3PToken(19)); Assert.AreEqual("2", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte))); }
private IInternalSession MockSession(string keyspace, bool coreEngine) { var keyspaces = new ConcurrentDictionary <string, KeyspaceMetadata>(); // unique configurations keyspaces.AddOrUpdate( keyspace, FakeSchemaParserFactory.CreateSimpleKeyspace(keyspace, 2, graphEngine: coreEngine ? "Core" : null), (s, keyspaceMetadata) => keyspaceMetadata); var schemaParser = new FakeSchemaParser(keyspaces); var config = new TestConfigurationBuilder { ConnectionFactory = new FakeConnectionFactory() }.Build(); var metadata = new Metadata(config, schemaParser) { Partitioner = "Murmur3Partitioner" }; var cluster = Mock.Of <ICluster>(); Mock.Get(cluster).SetupGet(c => c.Metadata).Returns(metadata); var session = Mock.Of <IInternalSession>(); Mock.Get(session).SetupGet(s => s.Cluster).Returns(cluster); metadata.ControlConnection = new ControlConnection( Mock.Of <IInternalCluster>(), new ProtocolEventDebouncer(new TaskBasedTimerFactory(), TimeSpan.FromMilliseconds(20), TimeSpan.FromSeconds(100)), ProtocolVersion.V3, config, metadata, new List <IContactPoint> { new IpLiteralContactPoint(IPAddress.Parse("127.0.0.1"), config.ProtocolOptions, config.ServerNameResolver) }); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.1"), 9042)); metadata.Hosts.First().SetInfo(new TestHelper.DictionaryBasedRow(new Dictionary <string, object> { { "data_center", "dc1" }, { "rack", "rack1" }, { "tokens", new [] { "100" } }, { "release_version", "3.11.1" } })); metadata.RebuildTokenMapAsync(false, true).GetAwaiter().GetResult(); Assert.IsNotNull(metadata.GetKeyspace(keyspace)); return(session); }
public void TokenMap_Build_SimpleStrategy_Adjacent_Ranges_Test() { var hosts = new[] { //0 and 100 are adjacent TestHelper.CreateHost("192.168.0.1", "dc1", "rack1", new HashSet <string> { "0", "100", "1000" }), TestHelper.CreateHost("192.168.0.2", "dc1", "rack1", new HashSet <string> { "200", "2000", "20000" }), TestHelper.CreateHost("192.168.0.3", "dc1", "rack1", new HashSet <string> { "300", "3000", "30000" }) }; var ks = FakeSchemaParserFactory.CreateSimpleKeyspace("ks1", 2); var map = TokenMap.Build("Murmur3Partitioner", hosts, new[] { ks }); var replicas = map.GetReplicas("ks1", new M3PToken(0)); Assert.AreEqual(2, replicas.Count); //It should contain the first host and the second, even though the first host contains adjacent CollectionAssert.AreEqual(new byte[] { 1, 2 }, replicas.Select(TestHelper.GetLastAddressByte)); }
public void RefreshSingleKeyspace_Should_BuildTokenMap_When_TokenMapIsNull() { var keyspaces = new ConcurrentDictionary <string, KeyspaceMetadata>(); keyspaces.GetOrAdd("ks1", FakeSchemaParserFactory.CreateSimpleKeyspace("ks1", 1)); var schemaParser = new FakeSchemaParser(keyspaces); var metadata = new Metadata(new Configuration(), schemaParser) { Partitioner = "Murmur3Partitioner" }; metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.1"), 9042));; metadata.Hosts.First().SetInfo(new TestHelper.DictionaryBasedRow(new Dictionary <string, object> { { "data_center", "dc1" }, { "rack", "rack1" }, { "tokens", GenerateTokens(1, 256) }, { "release_version", "3.11.1" } })); Assert.IsNull(metadata.TokenToReplicasMap); metadata.RefreshSingleKeyspace("ks1").GetAwaiter().GetResult(); Assert.NotNull(metadata.TokenToReplicasMap); }
public void Should_UpdateKeyspacesAndTokenMapCorrectly_When_MultipleThreadsCallingRefreshKeyspace() { var keyspaces = new ConcurrentDictionary <string, KeyspaceMetadata>(); // unique configurations keyspaces.AddOrUpdate("ks1", FakeSchemaParserFactory.CreateSimpleKeyspace("ks1", 2), (s, keyspaceMetadata) => keyspaceMetadata); keyspaces.AddOrUpdate("ks2", FakeSchemaParserFactory.CreateSimpleKeyspace("ks2", 10), (s, keyspaceMetadata) => keyspaceMetadata); keyspaces.AddOrUpdate("ks3", FakeSchemaParserFactory.CreateSimpleKeyspace("ks3", 5), (s, keyspaceMetadata) => keyspaceMetadata); keyspaces.AddOrUpdate("ks4", FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks4", new Dictionary <string, int> { { "dc1", 2 }, { "dc2", 2 } }), (s, keyspaceMetadata) => keyspaceMetadata); keyspaces.AddOrUpdate("ks5", FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks5", new Dictionary <string, int> { { "dc1", 1 }, { "dc2", 2 } }), (s, keyspaceMetadata) => keyspaceMetadata); keyspaces.AddOrUpdate("ks6", FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks6", new Dictionary <string, int> { { "dc1", 1 } }), (s, keyspaceMetadata) => keyspaceMetadata); // duplicate configurations keyspaces.AddOrUpdate("ks7", FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks7", new Dictionary <string, int> { { "dc1", 2 }, { "dc2", 2 } }), (s, keyspaceMetadata) => keyspaceMetadata); keyspaces.AddOrUpdate("ks8", FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks8", new Dictionary <string, int> { { "dc1", 1 } }), (s, keyspaceMetadata) => keyspaceMetadata); keyspaces.AddOrUpdate("ks9", FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks9", new Dictionary <string, int> { { "dc1", 1 }, { "dc2", 2 } }), (s, keyspaceMetadata) => keyspaceMetadata); keyspaces.AddOrUpdate("ks10", FakeSchemaParserFactory.CreateSimpleKeyspace("ks10", 10), (s, keyspaceMetadata) => keyspaceMetadata); keyspaces.AddOrUpdate("ks11", FakeSchemaParserFactory.CreateSimpleKeyspace("ks11", 2), (s, keyspaceMetadata) => keyspaceMetadata); var schemaParser = new FakeSchemaParser(keyspaces); var config = new TestConfigurationBuilder { ConnectionFactory = new FakeConnectionFactory() }.Build(); var metadata = new Metadata(config, schemaParser) { Partitioner = "Murmur3Partitioner" }; metadata.ControlConnection = new ControlConnection( Mock.Of <IInternalCluster>(), new ProtocolEventDebouncer(new TaskBasedTimerFactory(), TimeSpan.FromMilliseconds(20), TimeSpan.FromSeconds(100)), ProtocolVersion.V3, config, metadata, new List <IContactPoint> { new IpLiteralContactPoint(IPAddress.Parse("127.0.0.1"), config.ProtocolOptions, config.ServerNameResolver) }); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.1"), 9042)); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.2"), 9042)); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.3"), 9042)); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.4"), 9042)); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.5"), 9042)); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.6"), 9042)); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.7"), 9042)); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.8"), 9042)); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.9"), 9042)); metadata.Hosts.Add(new IPEndPoint(IPAddress.Parse("192.168.0.10"), 9042)); var initialToken = 1; foreach (var h in metadata.Hosts) { h.SetInfo(new TestHelper.DictionaryBasedRow(new Dictionary <string, object> { { "data_center", initialToken % 2 == 0 ? "dc1" : "dc2" }, { "rack", "rack1" }, { "tokens", GenerateTokens(initialToken, 256) }, { "release_version", "3.11.1" } })); initialToken++; } metadata.RebuildTokenMapAsync(false, true).GetAwaiter().GetResult(); var expectedTokenMap = metadata.TokenToReplicasMap; Assert.NotNull(expectedTokenMap); var bag = new ConcurrentBag <string>(); var tasks = new List <Task>(); for (var i = 0; i < 100; i++) { var index = i; tasks.Add(Task.Factory.StartNew( () => { for (var j = 0; j < 35; j++) { if (j % 10 == 0 && index % 2 == 0) { metadata.RefreshSchemaAsync().GetAwaiter().GetResult(); } else if (j % 16 == 0) { if (bag.TryTake(out var ksName)) { if (keyspaces.TryRemove(ksName, out var ks)) { metadata.RefreshSchemaAsync(ksName).GetAwaiter().GetResult(); ks = metadata.GetKeyspace(ksName); if (ks != null) { throw new Exception($"refresh for {ks.Name} returned non null after refresh single."); } } } } else if (j % 2 == 0) { if (bag.TryTake(out var ksName)) { if (keyspaces.TryRemove(ksName, out var ks)) { metadata.ControlConnection.HandleKeyspaceRefreshLaterAsync(ks.Name).GetAwaiter().GetResult(); ks = metadata.GetKeyspace(ksName); if (ks != null) { throw new Exception($"refresh for {ks.Name} returned non null after remove."); } } } } else { var keyspaceName = $"ks_____{index}_____{j}"; var ks = FakeSchemaParserFactory.CreateSimpleKeyspace(keyspaceName, (index * j) % 10); keyspaces.AddOrUpdate( keyspaceName, ks, (s, keyspaceMetadata) => ks); metadata.ControlConnection.HandleKeyspaceRefreshLaterAsync(ks.Name).GetAwaiter().GetResult(); ks = metadata.GetKeyspace(ks.Name); if (ks == null) { throw new Exception($"refresh for {keyspaceName} returned null after add."); } bag.Add(keyspaceName); } } }, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach)); } Task.WaitAll(tasks.ToArray()); AssertSameReplicas(keyspaces.Values, expectedTokenMap, metadata.TokenToReplicasMap); }
public void Build_Should_OnlyCallOncePerReplicationConfiguration_When_MultipleKeyspacesWithSameReplicationOptions() { var hosts = new List <Host> { { TestHelper.CreateHost("192.168.0.0", "dc1", "rack", new HashSet <string> { "0" }) }, { TestHelper.CreateHost("192.168.0.1", "dc1", "rack", new HashSet <string> { "10" }) }, { TestHelper.CreateHost("192.168.0.2", "dc1", "rack", new HashSet <string> { "20" }) }, { TestHelper.CreateHost("192.168.0.3", "dc2", "rack", new HashSet <string> { "30" }) }, { TestHelper.CreateHost("192.168.0.4", "dc2", "rack", new HashSet <string> { "40" }) } }; var factory = new ProxyReplicationStrategyFactory(); var keyspaces = new List <KeyspaceMetadata> { // unique configurations FakeSchemaParserFactory.CreateSimpleKeyspace("ks1", 2, factory), FakeSchemaParserFactory.CreateSimpleKeyspace("ks2", 10, factory), FakeSchemaParserFactory.CreateSimpleKeyspace("ks3", 5, factory), FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks4", new Dictionary <string, int> { { "dc1", 2 }, { "dc2", 2 } }, factory), FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks5", new Dictionary <string, int> { { "dc1", 1 }, { "dc2", 2 } }, factory), FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks6", new Dictionary <string, int> { { "dc1", 1 } }, factory), // duplicate configurations FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks7", new Dictionary <string, int> { { "dc1", 2 }, { "dc2", 2 } }, factory), FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks8", new Dictionary <string, int> { { "dc1", 1 } }, factory), FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks9", new Dictionary <string, int> { { "dc1", 1 }, { "dc2", 2 } }, factory), FakeSchemaParserFactory.CreateSimpleKeyspace("ks10", 10, factory), FakeSchemaParserFactory.CreateSimpleKeyspace("ks11", 2, factory) }; var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces); var proxyStrategies = keyspaces.Select(k => (ProxyReplicationStrategy)k.Strategy).ToList(); Assert.AreEqual(6, proxyStrategies.Count(strategy => strategy.Calls > 0)); AssertOnlyOneStrategyIsCalled(proxyStrategies, 0, 10); AssertOnlyOneStrategyIsCalled(proxyStrategies, 1, 9); AssertOnlyOneStrategyIsCalled(proxyStrategies, 2); AssertOnlyOneStrategyIsCalled(proxyStrategies, 3, 6); AssertOnlyOneStrategyIsCalled(proxyStrategies, 4, 8); AssertOnlyOneStrategyIsCalled(proxyStrategies, 5, 7); }
public void Build_Should_CopyReplicationStrategiesFromDictionaryToConcurrentDictionary_When_DifferentStrategiesAreUsed() { foreach (var rf1 in Enumerable.Range(1, 32).Select(r => r.ToString())) { foreach (var rf2 in Enumerable.Range(1, 32).Select(r => r.ToString())) { if (rf1 == rf2) { continue; } var keyspaces = new List <KeyspaceMetadata> { // unique configurations FakeSchemaParserFactory.CreateSimpleKeyspace("ks1", 2), FakeSchemaParserFactory.CreateSimpleKeyspace("ks2", 10), FakeSchemaParserFactory.CreateSimpleKeyspace("ks3", 5), FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks4", new Dictionary <string, string> { { "dc1", rf1 }, { "dc2", rf1 } }), FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks5", new Dictionary <string, string> { { "dc1", rf2 }, { "dc2", rf2 } }), FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks6", new Dictionary <string, string> { { "dc1", "1" } }), // duplicate configurations FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks7", new Dictionary <string, string> { { "dc1", rf1 }, { "dc2", rf1 } }), FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks8", new Dictionary <string, string> { { "dc1", "1" } }), FakeSchemaParserFactory.CreateNetworkTopologyKeyspace("ks9", new Dictionary <string, string> { { "dc1", rf2 }, { "dc2", rf2 } }), FakeSchemaParserFactory.CreateSimpleKeyspace("ks10", 10), FakeSchemaParserFactory.CreateSimpleKeyspace("ks11", 2) }; var strategies = keyspaces.Select(k => k.Strategy).ToList(); var dictionary = new Dictionary <IReplicationStrategy, object>(); foreach (var strategy in strategies) { if (!dictionary.ContainsKey(strategy)) { dictionary.Add(strategy, ""); } } // private const in ConcurrentDictionary const int defaultCapacity = 31; // would love to test every possible value but it would take too much time foreach (var concurrencyLevel in Enumerable.Range(1, 512)) { var concurrentDictionary = new ConcurrentDictionary <IReplicationStrategy, object>(concurrencyLevel, defaultCapacity); foreach (var strategy in dictionary) { if (!concurrentDictionary.TryAdd(strategy.Key, strategy.Value)) { Assert.Fail($"This would throw ArgumentException, duplicate values with processor count: {concurrencyLevel}, rf1: {rf1}, rf2: {rf2}"); return; } } } } } }