public void TestGetTwoOnlyTwoQuick()
{
var x = new ConsistentHash();
x.Add("abcdefg");
x.Add("hijklmn");
Quick.Check <string>(s =>
{
var(a, b) = x.GetTwo(s);
if (a == b)
{
return(false);
}
if (a != "abcdefg" && a != "hijklmn")
{
return(false);
}
if (b != "abcdefg" && b != "hijklmn")
{
return(false);
}
return(true);
});
}
public void TestGetMultipleRemove()
{
var x = new ConsistentHash();
x.Add("abcdefg");
x.Add("hijklmn");
x.Add("opqrstu");
foreach (var gmtest in new[]
{
// new[] { in, out }
new[] { "ggg", "abcdefg" },
new[] { "hhh", "opqrstu" },
new[] { "iiiii", "hijklmn" }
})
{
Assert.AreEqual(gmtest[1], x.Get(gmtest[0]));
}
x.Remove("hijklmn");
foreach (var gmtest in new[]
{
// new[] { in, out }
new[] { "ggg", "abcdefg" },
new[] { "hhh", "opqrstu" },
new[] { "iiiii", "opqrstu" }
})
{
Assert.AreEqual(gmtest[1], x.Get(gmtest[0]));
}
}
public void TestSet()
{
var x = new ConsistentHash();
x.Add("abc");
x.Add("def");
x.Add("ghi");
{
x.Set(new[] { "jkl", "mno" });
Assert.AreEqual(2, x.Count);
var(a, b) = x.GetTwo("qwerqwerwqer");
Assert.IsTrue(a == "jkl" || a == "mno");
Assert.IsTrue(b == "jkl" || b == "mno");
Assert.AreNotEqual(a, b);
}
{
x.Set(new[] { "pqr", "mno" });
Assert.AreEqual(2, x.Count);
var(a, b) = x.GetTwo("qwerqwerwqer");
Assert.IsTrue(a == "pqr" || a == "mno");
Assert.IsTrue(b == "pqr" || b == "mno");
Assert.AreNotEqual(a, b);
}
}
private static void Main(string[] args)
{
ConsistentHash<string> chash = new ConsistentHash<string>();
Console.WriteLine("Adding 10 servers.");
for (int i = 0; i < 10; i++) {
chash.Add("server"+i);
}
Console.WriteLine("Key distributions:");
var counts = new SortedDictionary<string, int>();
for (int i = 0; i < 1000; i++) {
var key = "key"+i;
var server = chash.GetNext(key);
if (!counts.ContainsKey(server)) {
counts[server] = 0;
}
counts[server]++;
}
foreach (var server in counts.Keys) {
Console.WriteLine("There are {0} keys assigned to {1}", counts[server], server);
}
Console.WriteLine("Adding 1 more server.");
chash.Add("serverX");
Console.WriteLine("New key distributions:");
counts.Clear();
for (int i = 0; i < 1000; i++) {
var key = "key"+i;
var server = chash.GetNext(key);
if (!counts.ContainsKey(server)) {
counts[server] = 0;
}
counts[server]++;
}
foreach (var server in counts.Keys) {
Console.WriteLine("There are {0} keys assigned to {1}", counts[server], server);
}
}
public void TestGetNLess()
{
var x = new ConsistentHash();
x.Add("abcdefg");
x.Add("hijklmn");
x.Add("opqrstu");
var members = x.GetN("99999999", 2).ToArray();
Assert.AreEqual(2, members.Length);
Assert.AreEqual("abcdefg", members[0]);
Assert.AreEqual("hijklmn", members[1]);
}
public void TestGetMultipleQuick()
{
var x = new ConsistentHash();
x.Add("abcdefg");
x.Add("hijklmn");
x.Add("opqrstu");
Quick.Check <string>(s =>
{
var y = x.Get(s);
return(y == "abcdefg" || y == "hijklmn" || y == "opqrstu");
});
}
public void TestGetTwo()
{
var x = new ConsistentHash();
x.Add("abcdefg");
x.Add("hijklmn");
x.Add("opqrstu");
var(a, b) = x.GetTwo("99999999");
Assert.AreNotEqual(a, b);
Assert.AreEqual("abcdefg", a);
Assert.AreEqual("hijklmn", b);
}
/// <summary>
/// 选择一个地址。
/// </summary>
/// <param name="context">地址选择上下文。</param>
/// <returns>地址模型。</returns>
protected override async Task <AddressModel> SelectAsync(AddressSelectContext context)
{
var key = GetCacheKey(context.Descriptor);
var addressEntry = _concurrent.GetOrAdd(key, k =>
{
var len = context.Address.Count();
len = len > 1 && len < 10 ? len * 10 : len;
var hash = new ConsistentHash <AddressModel>(_hashAlgorithm, len);
foreach (var address in context.Address)
{
hash.Add(address, address.ToString());
}
return(hash);
});
AddressModel addressModel;
var IsHealth = false;
do
{
addressModel = addressEntry.GetItemNode(context.Item);
IsHealth = await _healthCheckService.IsHealth(addressModel);
if (!IsHealth)
{
addressEntry.Remove(addressModel.ToString());
_unHealths.Add(new ValueTuple <string, AddressModel>(key, addressModel));
_healthCheckService.Changed += ItemNode_Changed;
}
} while (!IsHealth);
return(addressModel);
}
public void AddBulkPerformance()
{
var results = "";
foreach (HashProvider provider in Enum.GetValues(typeof(HashProvider)))
{
long itemCount = 0;
DateTime startTime = DateTime.Now;
var c = new ConsistentHash <HashableString>(provider);
while ((startTime + TestLength) > DateTime.Now)
{
var item = new HashableString(Guid.NewGuid().ToString());
item.ComputeHash(provider);
c.Add(item, false, 0);
itemCount++;
}
c.UpdateKeyArray();
results += GetResultString(provider, itemCount);
}
Assert.Pass(results);
}
static PartitionKeyService()
{
for (var i = 0; i < TOTAL_PARTITIONS; i++)
{
consistentHash.Add($"p{i}");
}
}
public void UpdateKeyArray()
{
var c = new ConsistentHash <HashableString>(HashProvider.SHA384);
var firstItem = new HashableString(Guid.NewGuid().ToString());
var secondItem = new HashableString(Guid.NewGuid().ToString());
firstItem.ComputeHash(c.Provider, null);
secondItem.ComputeHash(c.Provider, null);
c.Add(firstItem, true, 0);
c.Add(secondItem, false, 0);
Assert.False(c.ContainsNode(secondItem.ComputedHash));
c.UpdateKeyArray();
Assert.True(c.ContainsNode(firstItem.ComputedHash));
}
public void WeightedAddRemove()
{
int rounds = 1000;
var c = new ConsistentHash <HashableString>(HashProvider.SHA384);
var firstItem = new HashableString(Guid.NewGuid().ToString());
firstItem.ComputeHash(c.Provider, null);
c.Add(firstItem, true, rounds);
Assert.True(c.ContainsNode(firstItem.ComputedHash));
var rehashed = firstItem.ComputedHash.Rehash();
for (int i = 1; i < rounds; i++)
{
rehashed = firstItem.ComputedHash.Rehash();
Assert.True(c.ContainsNode(rehashed));
}
c.Remove(firstItem.ComputedHash, true);
Assert.False(c.ContainsNode(firstItem.ComputedHash));
Assert.True(c.NodeCount == 0);
rehashed = firstItem.ComputedHash.Rehash();
for (int i = 1; i < rounds; i++)
{
rehashed = firstItem.ComputedHash.Rehash();
Assert.False(c.ContainsNode(rehashed));
}
}
/// <summary>
/// 初始化设置哈希节点容器
/// </summary>
private void InitSettingHashStorage()
{
foreach (var dataContext in DataContextPool)
{
CacheTargetType targetType;
if (!Enum.TryParse(dataContext.Key, true, out targetType))
{
continue;
}
var hash = new ConsistentHash <ConsistentHashNode>(Ioc.Create <IHashAlgorithm>());
hash.Add(new ConsistentHashNode()
{
Type = targetType,
Host = dataContext.Value.Host,
Port = dataContext.Value.Port.ToString(),
UserName = dataContext.Value.UserName,
Password = dataContext.Value.Password,
MaxSize = dataContext.Value.MaxSize.ToString(),
MinSize = dataContext.Value.MinSize.ToString(),
Db = dataContext.Value.Db.ToString()
});
DicHash.GetOrAdd(targetType.ToString(), hash);
}
}
public void TestRemoveNonExisting()
{
var x = new ConsistentHash();
x.Add("abcdefg");
x.Remove("abcdefghijk");
Assert.AreEqual(20, x.Circle.Count);
}
public void TestAdd()
{
var x = new ConsistentHash();
x.Add("abcdefg");
Assert.AreEqual(20, x.Circle.Count);
Assert.AreEqual(20, x.SortedHashes.Length);
Assert.IsTrue(IsSorted(x.SortedHashes));
x.Add("qwer");
Assert.AreEqual(40, x.Circle.Count);
Assert.AreEqual(40, x.SortedHashes.Length);
Assert.IsTrue(IsSorted(x.SortedHashes));
}
public void TestRemove()
{
var x = new ConsistentHash();
x.Add("abcdefg");
x.Remove("abcdefg");
Assert.AreEqual(0, x.Circle.Count);
Assert.AreEqual(0, x.SortedHashes.Length);
}
public void AddOne()
{
var chash = new ConsistentHash<string>();
chash.Add("server1");
Assert.That(chash.GetNext("key1"), Is.EqualTo("server1"));
Assert.That(chash.GetNext("key2"), Is.EqualTo("server1"));
Assert.That(chash.GetNext("key3"), Is.EqualTo("server1"));
}
public void Test()
{
string[] array = { "AAA", "BBB", "CCC", "DDD" };
string[] array2 = { array[0], array[1], array[2] };
ConsistentHash<string> s = new ConsistentHash<string>(array);
s.Add(array[3]);
for(int i = 0; i < 100; i++)
{
Assert.Contains(s.GetNode(i.ToString()), array);
}
}
public void TestGetTwoOnlyOneInCircle()
{
var x = new ConsistentHash();
x.Add("abcdefg");
var(a, b) = x.GetTwo("99999999");
Assert.AreNotEqual(a, b);
Assert.AreEqual("abcdefg", a);
Assert.AreEqual(default(string), b);
}
public void TestGetSingle()
{
var x = new ConsistentHash();
x.Add("abcdefg");
Quick.Check <string>(s =>
{
var y = x.Get(s);
return(y == "abcdefg");
});
}
public void AddRemoveMultiple()
{
var chash = new ConsistentHash<string>();
chash.Add("server1");
chash.Add("server2");
chash.Add("server3");
Assert.Contains(chash.GetNext("key1"), new string[] {"server1", "server2", "server3"});
Assert.Contains(chash.GetNext("key2"), new string[] {"server1", "server2", "server3"});
Assert.Contains(chash.GetNext("key3"), new string[] {"server1", "server2", "server3"});
chash.Add("server4");
Assert.Contains(chash.GetNext("key1"), new string[] {"server1", "server2", "server3", "server4"});
Assert.Contains(chash.GetNext("key2"), new string[] {"server1", "server2", "server3", "server4"});
Assert.Contains(chash.GetNext("key3"), new string[] {"server1", "server2", "server3", "server4"});
Assert.Contains(chash.GetNext("key4"), new string[] {"server1", "server2", "server3", "server4"});
chash.Remove("server2");
Assert.Contains(chash.GetNext("key1"), new string[] {"server1", "server3", "server4"});
Assert.Contains(chash.GetNext("key2"), new string[] {"server1", "server3", "server4"});
Assert.Contains(chash.GetNext("key3"), new string[] {"server1", "server3", "server4"});
Assert.Contains(chash.GetNext("key4"), new string[] {"server1", "server3", "server4"});
chash.Remove("server1");
Assert.Contains(chash.GetNext("key1"), new string[] {"server3", "server4"});
Assert.Contains(chash.GetNext("key2"), new string[] {"server3", "server4"});
Assert.Contains(chash.GetNext("key3"), new string[] {"server3", "server4"});
Assert.Contains(chash.GetNext("key4"), new string[] {"server3", "server4"});
chash.Remove("server3");
Assert.Contains(chash.GetNext("key1"), new string[] {"server4"});
Assert.Contains(chash.GetNext("key2"), new string[] {"server4"});
Assert.Contains(chash.GetNext("key3"), new string[] {"server4"});
Assert.Contains(chash.GetNext("key4"), new string[] {"server4"});
}
public Hash RegisterPeer([FromForm] PeerInfo peer)
{
if (peer == null || peer.ComputedHash == null)
{
throw new ArgumentNullException("Peer info or peer cert was null");
}
peer.ComputeHash(provider);
peer.ReplicationWeight = Math.Min(peer.ReplicationWeight, ReplicationWeightMax);
peers.Add(peer, true, peer.ReplicationWeight);
return(peer.ComputedHash);
}
public void TestAddCollision()
{
// These two strings produce several crc32 collisions after "|i" is
// appended added by Consistent.eltKey.
const string s1 = "abear";
const string s2 = "solidiform";
var x = new ConsistentHash();
x.Add(s1);
x.Add(s2);
var elt1 = x.Get(s1);
var y = new ConsistentHash();
y.Add(s2);
y.Add(s1);
var elt2 = y.Get(s1);
Assert.AreEqual(elt1, elt2);
}
public void TestGetNMoreQuick()
{
var x = new ConsistentHash();
x.Add("abcdefg");
x.Add("hijklmn");
x.Add("opqrstu");
Quick.Check <string>(s =>
{
var members = x.GetN(s, 5).ToArray();
if (members.Length != 3)
{
return(false);
}
var set = new HashSet <string>();
foreach (var member in members)
{
if (set.Contains(member))
{
return(false);
}
set.Add(member);
if (member != "abcdefg" && member != "hijklmn" && member != "opqrstu")
{
return(false);
}
}
return(true);
});
}
public void UnweightedAddRemove()
{
var c = new ConsistentHash <HashableString>(HashProvider.SHA384);
var firstItem = new HashableString(Guid.NewGuid().ToString());
firstItem.ComputeHash(c.Provider, null);
c.Add(firstItem, true, 0);
Assert.True(c.ContainsNode(firstItem.ComputedHash));
Assert.True(c.NodeCount == 1);
c.Remove(firstItem);
Assert.False(c.ContainsNode(firstItem.ComputedHash));
Assert.True(c.NodeCount == 0);
}
public void TestAdd()
{
var c = new ConsistentHash();
c.Add("cacheA");
c.Add("cacheB");
c.Add("cacheC");
var users = new[] { "user_mcnulty", "user_bunk", "user_omar", "user_bunny", "user_stringer" };
Dump(users, c);
c.Add("cacheD");
c.Add("cacheE");
Dump(users, c);
c.Remove("cacheD");
c.Remove("cacheE");
Dump(users, c);
c.Remove("cacheC");
Dump(users, c);
}
/// <summary>
/// 初始化设置哈希节点容器
/// </summary>
/// <remarks>
/// <para>创建：范亮</para>
/// <para>日期：2016/4/2</para>
/// </remarks>
private void InitSettingHashStorage()
{
foreach (var dataContext in DataContextPool)
{
EventTargetType targetType;
if (!Sys.Enum.TryParse(dataContext.Key, true, out targetType))
{
continue;
}
var hash =
new ConsistentHash <ConsistentHashNode>(ServiceResolver.Current.GetService <IHashAlgorithm>());
dataContext.Value.ForEach(v =>
{
var db = "";
var dbs = v.Split(new string[] { "::" }, Sys.StringSplitOptions.RemoveEmptyEntries);
var server = v.Split('@');
var endpoints = server.Length > 1 ? server[1].Split(':') : server[0].Split(':');
var account = server.Length > 1 ? server[0].Split(':'): null;
var username = account != null && account.Length > 1 ? account[0] : null;
var password = server.Length > 1 ? account[account.Length - 1] : this._password;
if (endpoints.Length <= 1)
{
return;
}
if (dbs.Length > 1)
{
db = dbs[dbs.Length - 1];
}
var node = new ConsistentHashNode()
{
Type = targetType,
Host = endpoints[0],
Port = endpoints[1],
UserName = username,
Password = password,
UseRetryNum = this._useRetryNum,
CunsumerNum = this._cunsumerNum,
UseRateLimit = this._useRateLimit,
MaxSize = this._maxSize,
MinSize = this._minSize,
QueueName = db.ToString(CultureInfo.InvariantCulture)
};
hash.Add(node);
dicHash.GetOrAdd(targetType.ToString(), hash);
});
}
}
/// <summary>
/// 初始化设置哈希节点容器
/// </summary>
private void InitSettingHashStorage()
{
foreach (var dataContext in DataContextPool)
{
CacheTargetType targetType;
if (!Enum.TryParse(dataContext.Key, true, out targetType))
{
continue;
}
var hash =
new ConsistentHash <ConsistentHashNode>(_hashAlgorithm);
dataContext.Value.ForEach(v =>
{
var db = "";
var dbs = v.Split(new[] { "::" }, StringSplitOptions.RemoveEmptyEntries);
var server = v.Split('@');
var endpoints = server.Length > 1 ? server[1].Split(':') : server[0].Split(':');
var account = server.Length > 1 ? server[0].Split(':') : null;
var username = account != null && account.Length > 1 ? account[0] : null;
Debug.Assert(account != null, nameof(account) + " != null");
var password = server.Length > 1 ? account[account.Length - 1] : _password;
if (endpoints.Length <= 1)
{
return;
}
if (dbs.Length > 1)
{
db = dbs[dbs.Length - 1];
}
var node = new ConsistentHashNode
{
Type = targetType,
Host = endpoints[0],
Port = endpoints[1],
UserName = username,
Password = password,
MaxSize = _maxSize,
MinSize = _minSize,
Db = db.ToString()
};
hash.Add(node, string.Format("{0}:{1}", node.Host, node.Port));
dicHash.GetOrAdd(targetType.ToString(), hash);
});
}
}
public void NextNode()
{
var c = new ConsistentHash <HashableBytes>(HashProvider.SHA384);
var firstItem = new HashableBytes(new byte[] { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 });
var secondItem = new HashableBytes(new byte[] { 0x10, 0x10, 0x10, 0x10, 0x10, 0x10 });
var thirdItem = new HashableBytes(new byte[] { 0x20, 0x20, 0x20, 0x20, 0x20, 0x20 });
firstItem.ComputeHash(c.Provider, null);
secondItem.ComputeHash(c.Provider, null);
thirdItem.ComputeHash(c.Provider, null);
c.Add(secondItem, true, 0);
var founditem = c.GetNode(firstItem.ComputedHash);
Assert.True(secondItem == founditem);
var founditem2 = c.GetNode(thirdItem.ComputedHash);
Assert.True(secondItem == founditem2);
}
/// <summary>
/// 选择一个地址。
/// </summary>
/// <param name="context">地址选择上下文。</param>
/// <returns>地址模型。</returns>
protected override async Task <AddressModel> SelectAsync(AddressSelectContext context)
{
var key = GetCacheKey(context.Descriptor);
var addressEntry = _concurrent.GetOrAdd(key, k =>
{
var len = context.Address.Count();
len = len > 1 && len < 10 ? len * 10 : len;
var hash = new ConsistentHash <AddressModel>(_hashAlgorithm, len);
foreach (var address in context.Address)
{
hash.Add(address);
}
return(hash);
});
AddressModel addressModel;
do
{
addressModel = addressEntry.GetItemNode(context.Item);
} while (await _healthCheckService.IsHealth(addressModel) == false);
return(addressModel);
}
/// <summary>
/// 初始化设置哈希节点容器
/// </summary>
/// <remarks>
/// <para>创建：张宏伟</para>
/// <para>日期：2016/4/2</para>
/// </remarks>
private void InitSettingHashStorage()
{
foreach (var dataContext in DataContextPool)
{
var hash = new ConsistentHash <RedisEndPoint>(_hashAlgorithm);
dataContext.Value.ForEach(v =>
{
var db = "";
var dbs = v.Split(new string[] { "::" }, StringSplitOptions.RemoveEmptyEntries);
var server = v.Split('@');
var endpoints = server.Length > 1 ? server[1].Split(':') : server[0].Split(':');
var account = server.Length > 1 ? server[0].Split(':') : null;
var username = account != null && account.Length > 1 ? account[0] : null;
var password = server.Length > 1 ? account[account.Length - 1] : this._password;
if (endpoints.Length <= 1)
{
return;
}
if (dbs.Length > 1)
{
db = dbs[dbs.Length - 1];
}
var node = new RedisEndPoint()
{
Host = endpoints[0],
Port = int.Parse(endpoints[1]),
Password = password,
MaxSize = this._maxSize,
MinSize = this._minSize,
DbIndex = int.Parse(db) //数据库索引 127.0.0.1:2222::2 索引为2
};
hash.Add(node, string.Format("{0}:{1}", node.Host, node.Port));
dicHash.GetOrAdd(dataContext.Key, hash);
});
}
}
/// <summary>
/// redis数据上下文
/// </summary>
/// <param name="rule">规则</param>
/// <param name="args">参数</param>
public RedisContext(IHashAlgorithm hashAlgorithm, RedisCacheOptions options)
{
_hashAlgorithm = hashAlgorithm;
if (!string.IsNullOrWhiteSpace(options.Configuration))
{
string[] connectionString = options.Configuration.Split(',');
HashSet <string> connSet = new HashSet <string>();
for (int i = 0, j = connectionString.Length; i < j; i++)
{
var connStr = connectionString[i];
if (connSet.Contains(connStr))
{
continue;
}
connSet.Add(connStr);
var hash =
new ConsistentHash <ConsistentHashNode>(_hashAlgorithm);
var endpoints = connStr.Split(':');
hash.Add(new ConsistentHashNode()
{
Type = CacheTargetType.Redis,
Host = endpoints[0],
Port = endpoints[1],
UserName = options.InstanceName,
Password = options.Pwd,
MaxSize = this._maxSize,
MinSize = this._minSize,
Db = i.ToString()
});
dicHash.GetOrAdd(CacheTargetType.Redis.ToString(), hash);
}
}
dicHash = new ConcurrentDictionary <string, ConsistentHash <ConsistentHashNode> >();
}