public void TestMaxConnections()
{
var dbName = "test";
var tableName = "CreateClientPerRequest";
// make sure the db and table exists
Assert.IsNotNull(Utils.GetOrCreateTableAndDatabase(new Client(Utils.GetConfigNodes()), dbName, tableName));
Client.SetMaxConnections(10);
for (var i = 0; i < 11; i++)
{
Client client = new Client(Utils.GetConfigNodes());
client.SetGlobalTimeout(15000);
client.SetMasterTimeout(10000);
Database db = client.GetDatabase(dbName);
Table table = db.GetTable(tableName);
try
{
table.Set("" + i, "" + i);
client.Submit();
}
catch (SDBPException e)
{
Assert.IsTrue(i == 10 && e.Status == Status.SDBP_FAILURE);
}
}
}
internal Table(Client client, Database database, ulong tableID, string name)
{
this.client = client;
this.database = database;
this.tableID = tableID;
this.name = name;
}
public static void CheckDatabaseWithNumericKeys(string databaseName, string tableName)
{
var client = new Client(Utils.GetConfigNodes());
var jsonConfigState = client.GetJSONConfigState();
var configState = Utils.JsonDeserialize<ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState));
var shardServers = new List<ConfigState.ShardServer>();
Int64 tableID = 0;
foreach (var database in configState.databases)
{
if (database.name == databaseName)
{
foreach (var table in configState.tables)
{
if (table.databaseID == database.databaseID && table.name == tableName)
{
shardServers = ConfigStateHelpers.GetShardServersByTable(table, configState);
tableID = table.tableID;
break;
}
}
}
if (tableID != 0)
break;
}
Assert.IsTrue(tableID != 0);
ulong num = 50 * 1000 * 1000;
CompareNumericTableKeysHTTP(shardServers, tableID, num);
CompareNumericTableKeysBackwardsHTTP(shardServers, tableID, num);
}
public void CreateClientPerRequest()
{
var dbName = "test";
var tableName = "CreateClientPerRequest";
// make sure the db and table exists
Assert.IsNotNull(Utils.GetOrCreateTableAndDatabase(new Client(Config.GetNodes()), dbName, tableName));
// without connection pooling
Client.SetConnectionPoolSize(0);
for (var i = 0; i < 100 * 1000; i++)
{
Client client = new Client(Config.GetNodes());
Database db = client.GetDatabase(dbName);
Table table = db.GetTable(tableName);
table.Set("" + i, "" + i);
}
// with connection pooling
Client.SetConnectionPoolSize(100);
for (var i = 0; i < 100 * 1000; i++)
{
Client client = new Client(Config.GetNodes());
Database db = client.GetDatabase(dbName);
Table table = db.GetTable(tableName);
table.Set("" + i, "" + i);
}
}
public void ClientInactivityTest()
{
Client.SetTrace(true);
Client.SetLogFile("c:\\Users\\zszabo\\logs\\no_service_trace.txt");
string dbName = "test_db";
string tableName = "test_table";
Client client = new Client(Utils.GetConfigNodes());
Table tbl = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableName);
for (ulong i = 0; i < 1000; i++)
tbl.Set(Utils.Id(i), "test");
client.Submit();
client.SetGlobalTimeout(30000);
var timeout = 5 * client.GetGlobalTimeout();
Console.WriteLine("Now sleeping for " + timeout / 1000 + " seconds");
Thread.Sleep((int)timeout);
foreach (string key in tbl.GetKeyIterator(new StringRangeParams()))
Console.Write(key);
client.Close();
}
// test entry point
public static void Main(string[] args)
{
//Client.SetTrace(true);
string[] controllers = { "192.168.1.234:7080" };
Client client = new Client(controllers);
Database db = client.GetDatabase("test");
Table table = db.GetTable("test");
table.Set("foo", "foo");
System.Console.WriteLine(table.Get("foo"));
if (table.Get("bar") == null)
System.Console.WriteLine("null");
}
public static bool PrimaryShardServerHTTPAction(Client client, Quorum quorum, string action)
{
var configState = Utils.GetFullConfigState(client);
var shardServers = configState.shardServers;
var configQuorum = configState.quorums.First(cq => cq.quorumID == (long)quorum.QuorumID);
if (configQuorum.hasPrimary == false)
return false;
var shardServer = configState.shardServers.First(s => s.nodeID == configQuorum.primaryID);
var httpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
var response = Utils.HTTP.GET(Utils.HTTP.BuildUri(httpURI, action));
if (response == null)
return false;
return true;
}
public static bool RandomShardServerHTTPAction(Client client, Quorum quorum, string action)
{
var configState = Utils.GetFullConfigState(client);
var shardServers = configState.shardServers;
var configQuorum = configState.quorums.First(cq => cq.quorumID == (long)quorum.QuorumID);
var random = new Random();
// select shard server
var victimNodeID = configQuorum.activeNodes[random.Next(configQuorum.activeNodes.Count)];
foreach (var shardServer in shardServers)
{
if (shardServer.nodeID == victimNodeID)
{
var httpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
var response = Utils.HTTP.GET(Utils.HTTP.BuildUri(httpURI, action));
if (response == null)
return false;
return true;
}
}
return false;
}
public void CountBeforeSubmit()
{
string dbName = "test_db";
string tableName = "test_table";
Client client = new Client(Utils.GetConfigNodes());
Table tbl = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableName);
for(ulong i = 0; i < 1000; i++)
tbl.Set(Utils.Id(i), "test");
var cnt1 = tbl.Count(new ByteRangeParams());
client.Submit();
var cnt2 = tbl.Count(new StringRangeParams());
Console.WriteLine(cnt1 + " - " + cnt2);
Assert.IsTrue(cnt2 == 1000);
client.Close();
}
public void CreateAndCloseClients()
{
string dbName = "create_and_close_clients_db";
string tableName = "create_and_close_clients_table";
Client client1 = new Client(Utils.GetConfigNodes());
Client client2 = new Client(Utils.GetConfigNodes());
Table tbl1 = Utils.GetOrCreateEmptyTableAndDatabase(client1, dbName, tableName);
Table tbl2 = Utils.GetOrCreateEmptyTableAndDatabase(client2, dbName + "_2", tableName);
tbl1.TruncateTable();
//client.Submit();
tbl1.Get("0000000000001");
tbl1.Set("0000000000001", "test");
tbl1.Get("0000000000002");
tbl1.Set("0000000000002", "test");
client1.Submit();
tbl1.Get("0000000000001");
tbl1.Set("0000000000001", "test");
tbl1.Get("0000000000002");
tbl1.Set("0000000000002", "test");
client1.Submit();
tbl2.Get("0000000000001");
tbl2.Set("0000000000001", "test");
tbl2.Get("0000000000002");
tbl2.Set("0000000000002", "test");
//client1.Close();
//client2.Close();
}
public static void FixConsistency(Client client, List<ConfigState.ShardServer> shardServers, Int64 tableID, string startKey, string endKey)
{
if (shardServers.Count <= 1)
return;
var keyDiffs = new HashSet<string>();
var serverKeys = new string[shardServers.Count][];
var i = 0;
while (true)
{
System.Console.WriteLine("StartKey: " + startKey);
serverKeys = ConfigStateHelpers.ParallelFetchTableKeysHTTP(shardServers, tableID, startKey, endKey, true);
for (i = 1; i < serverKeys.Length; i++)
{
if (serverKeys[0].Length > 0 && serverKeys[i].Length > 1 &&
serverKeys[0].First().CompareTo(serverKeys[i].Last()) < 0)
{
foreach (var diff in serverKeys[i].Except(serverKeys[0]))
keyDiffs.Add(diff);
}
if (serverKeys[0].Length > 1 && serverKeys[i].Length > 0 &&
serverKeys[i].First().CompareTo(serverKeys[0].Last()) < 0)
{
foreach (var diff in serverKeys[0].Except(serverKeys[i]))
keyDiffs.Add(diff);
}
}
if (keyDiffs.Count != 0)
{
FixDiffs(client, shardServers, tableID, keyDiffs.ToList());
startKey = keyDiffs.Last();
keyDiffs = new HashSet<string>();
continue;
}
if (serverKeys[0].Length <= 1)
break;
startKey = serverKeys[0][serverKeys[0].Length - 1];
}
}
public void TestListTable()
{
var dbName = "Benchmark";
var tableName = "fileContentLockStartDateIndex";
Client client = new Client(Utils.GetConfigNodes());
Database db = client.GetDatabase(dbName);
Table tbl = db.GetTable(tableName);
var counter = 0;
foreach (var kv in tbl.GetKeyValueIterator(new ByteRangeParams()))
{
counter++;
}
client.Close();
}
public void TruncateAfterSet()
{
string dbName = "get_set_db";
string tableName = "get_set_db_table";
Client.SetLogFile("f:\\log.txt");
Client.SetTrace(true);
Client client = new Client(Utils.GetConfigNodes());
Table tbl = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableName);
try
{
tbl.Set("0000000000001", "test");
tbl.Set("0000000000002", "test");
tbl.TruncateTable();
client.Submit();
Assert.Fail("No SDBPException!");
}
catch (SDBPException)
{
}
//client.Close();
}
public static void ThreadFunc()
{
//Client.SetTrace(true);
//Client.SetLogFile("d:/out.txt");
//string[] controllers = { "127.0.0.1:7080" };
string[] controllers = { "192.168.137.110:7080", "192.168.137.111:7080", "192.168.137.112:7080" };
Client client = new Client(controllers);
Quorum quorum = client.GetQuorum("test");
Database db = client.GetDatabase("test");
Table test = db.GetTable("test");
Table indices = db.GetTable("indices");
Sequence IDs = indices.GetSequence("IDs");
//System.Console.WriteLine("Thread.CurrentThread.ManagedThreadId = {0}", Thread.CurrentThread.ManagedThreadId);
System.Random random = new System.Random(Thread.CurrentThread.ManagedThreadId);
var value = "";
while (value.Length < 10 * 1000)
value += "" + random.Next();
//using (client.Transaction(quorum, "foo" + random.Next(100)))
//{
// System.Console.WriteLine("Transaction started.");
// for (var j = 0; j < 10*1000; j++)
// table.Set("" + j, value);
// client.CommitTransaction();
// System.Console.WriteLine("Transaction finished.");
//}
//return;
ulong ID;
//for (var i = 0; i < 1000; i++)
while (true)
{
if (random.Next(3) == 0)
{
try
{
System.Console.WriteLine("Batch started.");
for (var j = 0; j < 10; j++)
{
if (random.Next(2) == 0)
test.Set("" + random.Next(1000 * 1000), value);
else
test.Delete("" + random.Next(1000 * 1000));
}
//for (var j = 0; j < 10 * 1000; j++)
// ID = IDs.GetNext;
client.Submit();
System.Console.WriteLine("Batch finished.");
}
catch (SDBPException)
{
System.Console.WriteLine("Batch failed.");
}
continue;
}
//if (random.Next(2) == 0)
//{
// try
// {
// System.Console.WriteLine("List started.");
// var count = 0;
// foreach (KeyValuePair<string, string> kv in test.GetKeyValueIterator(new StringRangeParams().Prefix("1")))
// {
// count++;
// //System.Console.WriteLine(kv.Key + " => " + kv.Value);
// if (count == 10)
// break;
// }
// System.Console.WriteLine("List finished.");
// }
// catch (SDBPException)
// {
// System.Console.WriteLine("List failed.");
// // why does this happen
// }
// continue;
//}
while (true)
{
try
{
using (client.Transaction(quorum, "foo" + random.Next(100)))
{
System.Console.WriteLine("Transaction started.");
for (var j = 0; j < 10; j++)
{
if (random.Next(2) == 0)
test.Set("" + random.Next(1000 * 1000), value);
else
test.Delete("" + random.Next(1000 * 1000));
}
//for (var j = 0; j < 1000; j++)
// ID = IDs.GetNext;
if (random.Next(2) == 0)
client.CommitTransaction();
else
client.RollbackTransaction();
System.Console.WriteLine("Transaction finished.");
break;
}
}
catch (SDBPException)
{
//System.Console.WriteLine("Exception.");
System.Threading.Thread.Sleep(1);
}
}
}
}
public void CheckConfigStateShardConsistency()
{
Console.WriteLine("\nChecking config state shard consistency...\n");
var client = new Client(Utils.GetConfigNodes());
var jsonConfigState = client.GetJSONConfigState();
var configState = Utils.JsonDeserialize<ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState));
foreach (var table in configState.tables)
{
System.Console.WriteLine("\nChecking table {0}", table.tableID);
var shards = ConfigStateHelpers.GetTableShards(table, configState.shards);
var shardMap = new Dictionary<string, ConfigState.Shard>();
// build a map of shards by firstkey
foreach (var shard in shards)
{
ConfigState.Shard testShard;
Assert.IsFalse(shardMap.TryGetValue(shard.firstKey, out testShard));
shardMap.Add(shard.firstKey, shard);
}
// walk the map of shards by the last key of the shard
var firstKey = "";
int count = 0;
ConfigState.Shard tmpShard;
while (shardMap.TryGetValue(firstKey, out tmpShard))
{
count += 1;
if (tmpShard.lastKey == "")
break;
firstKey = tmpShard.lastKey;
}
Assert.IsTrue(count == shards.Count);
}
}
public void ListTestsWithProxies()
{
var dbName = "test_db";
var tableName = "test_table";
int length = 10 * 1000;
uint num = 22 * 1000;
Client client = new Client(Utils.GetConfigNodes());
Table tbl = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableName);
var value = Utils.RandomString(length);
for (ulong i = 0; i < num; i++)
tbl.Set(Utils.Id(i), value);
// don't Submit, proxied values
//client.Submit();
ListTests(tbl, num);
}
public static void FillDatabaseWithNumericKeys(string databaseName, string tableName)
{
var client = new Client(Utils.GetConfigNodes());
Assert.IsTrue(ConfigStateHelpers.TryDeleteDatabase(client, databaseName));
var database = client.CreateDatabase(databaseName);
Assert.IsNotNull(database, "Cannot create database " + databaseName);
var table = ConfigStateHelpers.TryCreateTable(database, tableName);
Assert.IsNotNull(table, "Cannot create table " + tableName);
System.Console.WriteLine("Filling the database...");
DateTime last = DateTime.Now;
for (ulong i = 0; i < 100 * 1000 * 1000; i++)
{
var key = Utils.Id(i);
table.Set(key, key);
TimeSpan timeSpan = DateTime.Now - last;
if (timeSpan.TotalSeconds >= 60)
{
System.Console.WriteLine("i: " + i);
last = DateTime.Now;
}
}
client.Submit();
System.Console.WriteLine("Database filling is done.");
}
public void ListTestsWithoutProxies()
{
var dbName = "test_db";
var tableName = "test_table";
int length = 10 * 1000;
uint num = 22 * 1000;
Client client = new Client(Utils.GetConfigNodes());
Table tbl = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableName);
var value = Utils.RandomString(length);
for (ulong i = 0; i < num; i++)
tbl.Set(Utils.Id(i), value);
// Submit, no proxied values left
client.Submit();
client.SetConsistencyMode(Client.CONSISTENCY_STRICT);
ListTests(tbl, num);
client.SetConsistencyMode(Client.CONSISTENCY_RYW);
ListTests(tbl, num);
}
public static void FixDiffs(Client client, List<ConfigState.ShardServer> shardServers, Int64 tableID, List<string> diffs)
{
var i = 0;
foreach (var key in diffs)
{
i += 1;
byte[] startKey = Utils.StringToByteArray(key);
byte[] endKey = Utils.NextKey(startKey);
var serverKeyValues = ConfigStateHelpers.ParallelFetchTableKeyValuesHTTP(shardServers, tableID, startKey, endKey, true);
if (Array.TrueForAll(serverKeyValues, val => (val.Count == 1 && Utils.ByteArraysEqual(val.First().Key, serverKeyValues[0].First().Key))))
continue;
foreach (var keyValue in serverKeyValues)
{
if (keyValue == null || keyValue.Count == 0)
continue;
if (keyValue.First().Value.Length > 0)
{
Assert.IsTrue(Utils.ByteArraysEqual(Utils.StringToByteArray(key), keyValue.First().Key));
Console.WriteLine("Setting key {0}", key);
Table table = new Table(client, null, (ulong)tableID, "");
table.Set(startKey, keyValue.First().Value);
client.Submit();
}
}
}
}
public void ListTestLinq()
{
Client client = new Client(Utils.GetConfigNodes());
Table table = client.GetDatabase("Benchmark").GetTable("transactionNetworkTransaction");
byte[] prefix = Utils.StringToByteArray("N:0000000000000|T:000000");
byte[] startKey = Utils.StringToByteArray("N:0000000000000|T:0000001250847|");
var rangeParams = new ByteRangeParams().StartKey(startKey).Prefix(prefix);
ByteKeyValueIterator iterator = table.GetKeyValueIterator(rangeParams);
var list = new List<KeyValuePair<byte[], byte[]>>();
list.AddRange(iterator.Select(kv => kv));
}
public void CheckClusterConsistencyByKeyValues()
{
var client = new Client(Utils.GetConfigNodes());
var jsonConfigState = client.GetJSONConfigState();
var configState = Utils.JsonDeserialize<ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState));
foreach (var table in configState.tables)
{
var shards = ConfigStateHelpers.GetTableShards(table, configState.shards);
foreach (var shard in shards)
{
System.Console.WriteLine("\nComparing shard key-values on shard " + shard.shardID);
var tableID = shard.tableID;
var startKey = Utils.StringToByteArray(shard.firstKey);
var endKey = Utils.StringToByteArray(shard.lastKey);
var quorum = ConfigStateHelpers.GetQuorum(configState, shard.quorumID);
var shardServers = ConfigStateHelpers.GetQuorumActiveShardServers(configState, quorum);
CompareTableKeyValuesHTTP(shardServers, tableID, startKey, endKey);
System.Console.WriteLine("Shard " + shard.shardID + " is consistent.\n");
}
}
}
public void GetSetSubmit()
{
string dbName = "get_set_db";
string tableName = "get_set_db_table";
Client client = new Client(Utils.GetConfigNodes());
Table tbl = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableName);
//client.Submit();
tbl.Get("0000000000001");
tbl.Set("0000000000001", "test");
tbl.Get("0000000000002");
tbl.Set("0000000000002", "test");
client.Submit();
var i = tbl.Count(new ByteRangeParams());
Assert.IsTrue(i == 2);
}
public static Database TryCreateDatabase(Client client, string databaseName)
{
try
{
var database = client.CreateDatabase(databaseName);
return database;
}
catch (SDBPException)
{
return null;
}
}
public void SequenceAddAfterSet()
{
string dbName = "seq_and_set_db";
string tableNameSeq = "seq_table";
string tableNameSet = "set_table";
Client client = new Client(Utils.GetConfigNodes());
Table tblSeq = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableNameSeq);
Table tblSet = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableNameSet);
Sequence testSeq = tblSeq.GetSequence("seqIDs");
tblSet.Set("0000000000001", "test");
tblSet.Set("0000000000002", "test");
tblSet.Set("0000000000003", "test");
Console.WriteLine("Next: " + testSeq.GetNext);
client.Submit();
client.Close();
}
public static bool TryDeleteDatabase(Client client, string databaseName)
{
try
{
var database = client.GetDatabase(databaseName);
try
{
database.DeleteDatabase();
return true;
}
catch (SDBPException)
{
return false;
}
}
catch (SDBPException)
{
return true;
}
}
public void SequenceResetAndTruncate()
{
string dbName = "seq_and_trunc_db";
string tableNameSeq = "seq_table";
string tableNameTrunc = "trunc_table";
Client client = new Client(Utils.GetConfigNodes());
Table tblSeq = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableNameSeq);
Table tblTrunc = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableNameTrunc);
Sequence testSeq = tblSeq.GetSequence("seqIDs");
try
{
testSeq.Reset();
tblTrunc.TruncateTable();
client.Submit();
Assert.Fail("No SDBPException!");
}
catch (SDBPException)
{
}
//client.Close();
}
internal Rollbacker(Client client, Quorum quorum, byte[] majorKey)
{
this.client = client;
client.StartTransaction(quorum, majorKey);
}
//[TestMethod]
public void SetGetMP3()
{
string dbName = "test_mp3_db";
string tableName = "test_mp3_table";
Client client = new Client(Utils.GetConfigNodes());
Table tbl = Utils.GetOrCreateEmptyTableAndDatabase(client, dbName, tableName);
// SET MP3 (you will need a valdi path to test.mp3
byte[] payload = Utils.ReadFile("f:/test.mp3");
System.Console.WriteLine("mp3 buffer: {0}", payload.GetLength(0));
tbl.Set(System.Text.Encoding.UTF8.GetBytes("mp3"), payload);
client.Submit();
byte[] res = tbl.Get(System.Text.Encoding.UTF8.GetBytes("mp3"));
System.Console.WriteLine("mp3 buffer: {0}", res.GetLength(0));
Assert.IsTrue(Utils.ByteArraysEqual(payload, res));
}
internal Quorum(Client client, ulong quorumID, string name)
{
this.client = client;
this.quorumID = quorumID;
this.name = name;
}
public void CheckConfigStateConsistency()
{
Console.WriteLine("\nChecking config state consistency...\n");
var client = new Client(Utils.GetConfigNodes());
var jsonConfigState = client.GetJSONConfigState();
var clientConfigState = Utils.JsonDeserialize<ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState));
var master = clientConfigState.master;
var controllers = GetControllersHTTPEndpoint(Utils.GetConfigNodes());
foreach (var controller in controllers)
{
var url = controller + "json/getconfigstate";
jsonConfigState = Utils.HTTP.GET(url, COUNT_TIMEOUT);
var configState = Utils.JsonDeserialize<ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState));
Assert.IsTrue(configState.master == -1 || configState.master == master);
}
}