public void Writer_ProcessDataFromDistributor_SendResultBack()
{
const int distrServer1 = 22180;
const int distrServer12 = 23180;
const int storageServer1 = 22181;
var writer =
new HashWriter(new HashMapConfiguration("TestTransaction1D1S", HashMapCreationMode.CreateNew, 1, 1,
HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", storageServer1, 157);
writer.Save();
_distributor1.Build(1, distrServer1, distrServer12, "TestTransaction1D1S");
_writer1.Build(storageServer1, "TestTransaction1D1S", 1);
_distributor1.Start();
_writer1.Start();
var list = new List <InnerData>();
const int count = 100;
for (int i = 1; i < count + 1; i++)
{
var ev =
new InnerData(new Transaction(HashConvertor.GetString(i.ToString(CultureInfo.InvariantCulture)), "")
{
OperationName = OperationName.Create,
OperationType = OperationType.Async
})
{
Data = CommonDataSerializer.Serialize(i),
Key = CommonDataSerializer.Serialize(i),
Transaction = { TableName = "Int" }
};
list.Add(ev);
}
foreach (var data in list)
{
_distributor1.Input.ProcessAsync(data);
}
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
foreach (var data in list)
{
var transaction = _distributor1.Main.GetTransactionState(data.Transaction.UserTransaction);
Assert.AreEqual(TransactionState.Complete, transaction.State);
}
var mem = _writer1.Db.GetDbModules.First() as TestDbInMemory;
Assert.AreEqual(count, mem.Local);
_writer1.Dispose();
_distributor1.Dispose();
}
public void AddServer(ServerId server)
{
_lock.EnterWriteLock();
if (!_model.Exists(x => x.Equals(server)))
{
var hash = HashConvertor.GetString(server.ToString());
_model.Add(new DistributorDescription(hash, server));
}
_lock.ExitWriteLock();
}
public void AddServers(List <ServerId> servers)
{
_lock.EnterWriteLock();
foreach (var server in servers)
{
if (!_model.Exists(x => x.Equals(server)))
{
var hash = HashConvertor.GetString(server.ToString());
_model.Add(new DistributorDescription(hash, server));
}
}
_lock.ExitWriteLock();
}
public void Writer_ProcessData_SendResultToDistributerMock()
{
const int distributorServer1 = 22171;
const int storageServer1 = 22172;
var writer =
new HashWriter(new HashMapConfiguration("TestDbTransaction", HashMapCreationMode.CreateNew, 1, 1,
HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", storageServer1, 157);
writer.Save();
_writer1.Build(storageServer1, "TestDbTransaction", 1);
_writer1.Start();
var s = TestHelper.OpenDistributorHostForDb(new ServerId("localhost", distributorServer1),
new ConnectionConfiguration("testService", 10));
var list = new List <InnerData>();
const int count = 100;
for (int i = 1; i < count + 1; i++)
{
var ev =
new InnerData(new Transaction(HashConvertor.GetString(i.ToString(CultureInfo.InvariantCulture)), "")
{
OperationName = OperationName.Create
})
{
Data = CommonDataSerializer.Serialize(i),
Key = CommonDataSerializer.Serialize(i),
Transaction = { Distributor = new ServerId("localhost", distributorServer1) }
};
ev.Transaction.TableName = "Int";
list.Add(ev);
}
foreach (var data in list)
{
_writer1.Q.DbInputProcessQueue.Add(data);
}
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
Assert.AreEqual(count, s.SendValue);
_writer1.Dispose();
}
public void WriterSystemModel_GetDestination_ChechAvailableServers()
{
var config = new DistributorHashConfiguration(1);
var writer = new HashWriter(new HashMapConfiguration("test", HashMapCreationMode.CreateNew, 6, 3, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "local", 11010, 157);
writer.SetServer(1, "local", 11011, 157);
writer.SetServer(2, "local", 11012, 157);
writer.SetServer(3, "local", 11010, 157);
writer.SetServer(4, "local", 11011, 157);
writer.SetServer(5, "local", 11012, 157);
writer.Save();
var model = new WriterSystemModel(config,
new HashMapConfiguration("test", HashMapCreationMode.ReadFromFile, 1,
1, HashFileType.Distributor));
model.Start();
var ev = new InnerData(new Transaction("123", ""))
{
Transaction = new Transaction(HashConvertor.GetString("1"), "")
};
ev.Transaction.Destination = new List <ServerId>();
var ret = model.GetDestination(ev);
Assert.IsTrue(ret.Count == 1);
model.ServerNotAvailable(ret.First());
var ret2 = model.GetDestination(ev);
Assert.IsTrue(ret2.Count == 1);
Assert.AreNotEqual(ret.First(), ret2.First());
model.ServerNotAvailable(ret2.First());
var ret3 = model.GetDestination(ev);
Assert.IsTrue(ret3.Count == 1);
Assert.AreNotEqual(ret.First(), ret3.First());
Assert.AreNotEqual(ret3.First(), ret2.First());
model.ServerNotAvailable(ret3.First());
var ret4 = model.GetDestination(ev);
Assert.IsTrue(ret4.Count == 0);
}
public void WriterSystemModel_GetDestination_CountReplics()
{
var config = new DistributorHashConfiguration(4);
var writer = new HashWriter(new HashMapConfiguration("testhash", HashMapCreationMode.CreateNew, 6, 3, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "local", 11010, 157);
writer.SetServer(1, "local", 11011, 157);
writer.SetServer(2, "local", 11012, 157);
writer.SetServer(3, "local", 11010, 157);
writer.SetServer(4, "local", 11011, 157);
writer.SetServer(5, "local", 11012, 157);
writer.Save();
var model = new WriterSystemModel(config,
new HashMapConfiguration("testhash", HashMapCreationMode.ReadFromFile,
1, 1, HashFileType.Distributor));
model.Start();
var ev = new InnerData(new Transaction("123", ""))
{
Transaction = new Transaction(HashConvertor.GetString("1"), "")
};
ev.Transaction.Destination = new List <ServerId>();
var ret = model.GetDestination(ev);
Assert.IsTrue(ret.Count == 0);
model = new WriterSystemModel(new DistributorHashConfiguration(3),
new HashMapConfiguration("testhash", HashMapCreationMode.ReadFromFile, 1,
1, HashFileType.Distributor));
model.Start();
ret = model.GetDestination(ev);
Assert.AreEqual(3, ret.Count);
Assert.AreNotEqual(ret[0], ret[1]);
Assert.AreNotEqual(ret[0], ret[2]);
Assert.AreNotEqual(ret[2], ret[1]);
}
public string CalculateHashFromValue(object value)
{
return(HashConvertor.GetString(_dataProvider.CalculateHashFromValue((TValue)value)));
}
public string CalculateHashFromKey(object key)
{
return(HashConvertor.GetString(_dataProvider.CalculateHashFromKey((TKey)key)));
}
public string CalculateHashFromValue(int value)
{
return(HashConvertor.GetString(value.ToString(CultureInfo.InvariantCulture)));
}
public string CalculateHashFromKey(int key)
{
return(HashConvertor.GetString(key.ToString(CultureInfo.InvariantCulture)));
}
public void Writer_Restore_TwoServers()
{
var writer =
new HashWriter(new HashMapConfiguration("test8", HashMapCreationMode.CreateNew, 2, 3,
HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", storageServer1, 157);
writer.SetServer(1, "localhost", storageServer2, 157);
writer.Save();
_distrTest.Build(1, distrServer1, distrServer12, "test8");
_writer1.Build(storageServer1, "test8", 1);
_writer2.Build(storageServer2, "test8", 1);
_distrTest.Start();
_writer1.Start();
var list = new List <InnerData>();
const int count = 1;
for (int i = 1; i < count + 1; i++)
{
var ev =
new InnerData(new Transaction(HashConvertor.GetString(i.ToString(CultureInfo.InvariantCulture)), "")
{
OperationName = OperationName.Create,
OperationType = OperationType.Async
})
{
Data = CommonDataSerializer.Serialize(i),
Key = CommonDataSerializer.Serialize(i),
Transaction = { Distributor = new ServerId("localhost", distrServer1) }
};
ev.Transaction.TableName = "Int";
list.Add(ev);
}
foreach (var data in list)
{
_distrTest.Input.ProcessAsync(data);
}
Thread.Sleep(TimeSpan.FromMilliseconds(6000));
foreach (var data in list)
{
var tr = _distrTest.Main.GetTransactionState(data.Transaction.UserTransaction);
if (tr.State != TransactionState.Complete)
{
data.Transaction = new Transaction(data.Transaction);
data.Transaction.ClearError();
_distrTest.Input.ProcessAsync(data);
}
}
Thread.Sleep(TimeSpan.FromMilliseconds(5000));
var mem = _writer1.Db.GetDbModules.First() as TestDbInMemory;
var mem2 = _writer2.Db.GetDbModules.First() as TestDbInMemory;
if (count > 1)
{
Assert.AreNotEqual(count, mem.Local);
Assert.AreNotEqual(count, mem.Remote);
}
Assert.AreEqual(count, mem.Local + mem.Remote);
_writer2.Start();
_writer2.Distributor.Restore(new ServerId("localhost", distrServer1), false);
Thread.Sleep(TimeSpan.FromMilliseconds(2000));
Assert.AreEqual(0, mem.Remote);
Assert.AreEqual(0, mem2.Remote);
Assert.AreEqual(count, mem.Local + mem2.Local);
Assert.AreEqual(false, _writer1.Restore.IsNeedRestore);
Assert.AreEqual(false, _writer2.Restore.IsNeedRestore);
_distrTest.Dispose();
_writer1.Dispose();
_writer2.Dispose();
}
public void Writer_RestoreAfterUpdateHashFile_ThreeServers()
{
var writer =
new HashWriter(new HashMapConfiguration("TestRestore3ServersUpdate", HashMapCreationMode.CreateNew, 2, 3,
HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", storageServer1, 157);
writer.SetServer(1, "localhost", storageServer2, 157);
writer.Save();
_distrTest.Build(1, distrServer1, distrServer12, "TestRestore3ServersUpdate");
_writer1.Build(storageServer1, "TestRestore3ServersUpdate", 1);
_writer2.Build(storageServer2, "TestRestore3ServersUpdate", 1);
_writer3.Build(storageServer3, "TestRestore3ServersUpdate", 1);
_distrTest.Start();
_writer1.Start();
_writer2.Start();
#region hell
var list = new List <InnerData>();
const int count = 50;
for (int i = 1; i < count + 1; i++)
{
var ev =
new InnerData(new Transaction(HashConvertor.GetString(i.ToString(CultureInfo.InvariantCulture)), "")
{
OperationName = OperationName.Create,
OperationType = OperationType.Async
})
{
Data = CommonDataSerializer.Serialize(i),
Key = CommonDataSerializer.Serialize(i),
Transaction = { Distributor = new ServerId("localhost", distrServer1) }
};
ev.Transaction.TableName = "Int";
list.Add(ev);
}
foreach (var data in list)
{
_distrTest.Input.ProcessAsync(data);
}
Thread.Sleep(TimeSpan.FromMilliseconds(5000));
#endregion
var mem = _writer1.Db.GetDbModules.First() as TestDbInMemory;
var mem2 = _writer2.Db.GetDbModules.First() as TestDbInMemory;
var mem3 = _writer3.Db.GetDbModules.First() as TestDbInMemory;
if (count > 1)
{
Assert.AreNotEqual(count, mem.Local);
Assert.AreNotEqual(count, mem.Remote);
Assert.AreNotEqual(count, mem2.Local);
Assert.AreNotEqual(count, mem2.Remote);
}
Assert.AreEqual(count, mem.Local + mem2.Local);
Assert.AreEqual(0, mem.Remote);
Assert.AreEqual(0, mem2.Remote);
writer =
new HashWriter(new HashMapConfiguration("TestRestore3ServersUpdate", HashMapCreationMode.CreateNew, 3, 3,
HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", storageServer1, 157);
writer.SetServer(1, "localhost", storageServer2, 157);
writer.SetServer(2, "localhost", storageServer3, 157);
writer.Save();
_writer3.Start();
_distrTest.Distributor.UpdateModel();
_writer1.Distributor.UpdateModel();
_writer2.Distributor.UpdateModel();
_writer3.Distributor.Restore(new ServerId("localhost", distrServer1), true);
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
Assert.AreEqual(0, mem.Remote);
Assert.AreEqual(0, mem2.Remote);
Assert.AreEqual(0, mem3.Remote);
Assert.AreNotEqual(0, mem3.Local);
Assert.AreEqual(count, mem.Local + mem2.Local + mem3.Local);
Assert.AreEqual(false, _writer1.Restore.IsNeedRestore);
Assert.AreEqual(false, _writer2.Restore.IsNeedRestore);
Assert.AreEqual(false, _writer3.Restore.IsNeedRestore);
_writer1.Dispose();
_writer2.Dispose();
_writer3.Dispose();
_distrTest.Dispose();
}
public void InputModuleWithParallel_ProcessAsync_SendToOneServersAndTimeoutInCache_Success()
{
const int distrServer1 = 22166;
const int distrServer2 = 23166;
const int storageServer1 = 22167;
var q1 = new GlobalQueueInner();
var writer =
new HashWriter(new HashMapConfiguration("testAsyncTrans1S", HashMapCreationMode.CreateNew, 1, 1, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", storageServer1, 157);
writer.Save();
#region hell
GlobalQueue.SetQueue(q1);
var connection = new ConnectionConfiguration("testService", 10);
var distrconfig = new DistributorHashConfiguration(1);
var queueconfig = new QueueConfiguration(1, 100);
var dnet = new DistributorNetModule(connection,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var ddistributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
queueconfig, dnet,
new ServerId("localhost", distrServer1),
new ServerId("localhost", distrServer2),
new HashMapConfiguration("testAsyncTrans1S",
HashMapCreationMode.ReadFromFile,
1, 1, HashFileType.Distributor));
dnet.SetDistributor(ddistributor);
var tranc = new TransactionModule(new QueueConfiguration(1, 1000), dnet, new TransactionConfiguration(4),
distrconfig);
var cache = new DistributorTimeoutCache(TimeSpan.FromMilliseconds(2000), TimeSpan.FromMilliseconds(200000));
var main = new MainLogicModule(cache, ddistributor, tranc);
cache.SetMainLogicModule(main);
var netReceive4 = new NetReceiverConfiguration(distrServer1, "localhost", "testService");
var netReceive42 = new NetReceiverConfiguration(distrServer2, "localhost", "testService");
var input = new InputModuleWithParallel(new QueueConfiguration(2, 100000), main, tranc);
var receiver4 = new NetDistributorReceiver(main, input, ddistributor, netReceive4, netReceive42);
#endregion
var s = TestHelper.OpenWriterHost(new ServerId("localhost", storageServer1),
new ConnectionConfiguration("testService", 10));
main.Start();
receiver4.Start();
input.Start();
dnet.Start();
ddistributor.Start();
q1.Start();
var list = new List <InnerData>();
const int count = 1;
for (int i = 1; i < count + 1; i++)
{
var ev =
new InnerData(new Transaction(HashConvertor.GetString(i.ToString(CultureInfo.InvariantCulture)), "")
{
OperationName = OperationName.Create,
OperationType = OperationType.Async
})
{
Data = CommonDataSerializer.Serialize(i)
};
list.Add(ev);
}
foreach (var data in list)
{
input.ProcessAsync(data);
}
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
Assert.AreEqual(count, s.Value);
foreach (var data in list)
{
var transaction = main.GetTransactionState(data.Transaction.UserTransaction);
Assert.AreEqual(TransactionState.TransactionInProcess, transaction.State);
}
Thread.Sleep(TimeSpan.FromMilliseconds(2000));
foreach (var data in list)
{
var transaction = main.GetTransactionState(data.Transaction.UserTransaction);
if (transaction.State == TransactionState.DontExist)
{
Thread.Sleep(1000);
}
transaction = main.GetTransactionState(data.Transaction.UserTransaction);
Assert.AreEqual(TransactionState.Error, transaction.State);
}
q1.Dispose();
ddistributor.Dispose();
dnet.Dispose();
cache.Dispose();
}
public void MainLogic_ProcessWithData_SendAllReplicsThenObsoleteDataInCache()
{
var writer = new HashWriter(new HashMapConfiguration("test9", HashMapCreationMode.CreateNew, 2, 3, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", 21111, 157);
writer.SetServer(1, "localhost", 21112, 157);
writer.Save();
var cache = new DistributorTimeoutCache(TimeSpan.FromMilliseconds(400), TimeSpan.FromMilliseconds(1000));
var distrconfig = new DistributorHashConfiguration(2);
var queueconfig = new QueueConfiguration(1, 100);
var netconfig = new ConnectionConfiguration("testService", 10);
var net = new DistributorNetModule(netconfig,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var distributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
queueconfig, net, new ServerId("localhost", 1),
new ServerId("localhost", 1),
new HashMapConfiguration("test9", HashMapCreationMode.ReadFromFile,
1, 1, HashFileType.Distributor));
net.SetDistributor(distributor);
var transaction = new TransactionModule(new QueueConfiguration(1, 1000), net, new TransactionConfiguration(1),
distrconfig);
var main = new MainLogicModule(cache, distributor, transaction);
cache.SetMainLogicModule(main);
var server1 = new ServerId("localhost", 21111);
var server2 = new ServerId("localhost", 21112);
var s1 = TestHelper.OpenWriterHost(server1, netconfig);
var s2 = TestHelper.OpenWriterHost(server2, netconfig);
cache.Start();
distributor.Start();
net.Start();
transaction.Start();
main.Start();
GlobalQueue.Queue.Start();
net.ConnectToWriter(server1);
net.ConnectToWriter(server2);
Thread.Sleep(TimeSpan.FromMilliseconds(300));
var ev = new InnerData(new Transaction("123", ""))
{
Transaction =
new Transaction(HashConvertor.GetString("1"), "")
{
OperationName = OperationName.Create,
OperationType = OperationType.Async
}
};
ev.Transaction.Destination = new List <ServerId>();
using (var trans = transaction.Rent())
{
main.ProcessWithData(ev, trans.Element);
}
GlobalQueue.Queue.TransactionQueue.Add(ev.Transaction);
GlobalQueue.Queue.TransactionQueue.Add(ev.Transaction);
Thread.Sleep(TimeSpan.FromMilliseconds(100));
Assert.IsTrue(s1.Value > 0);
Assert.IsTrue(s2.Value > 0);
Assert.AreEqual(main.GetTransactionState(ev.Transaction.UserTransaction).State, TransactionState.Complete);
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
Assert.AreEqual(main.GetTransactionState(ev.Transaction.UserTransaction).State, TransactionState.DontExist);
net.Dispose();
distributor.Dispose();
transaction.Dispose();
}
public string CalculateHashFromValue(object value)
{
return(HashConvertor.GetString(value.ToString()));
}
public string CalculateHashFromKey(object key)
{
return(HashConvertor.GetString(key.ToString()));
}