public void Build(int countReplics, int distrServer1, int distrServer12, string hashFile)
{
_q = new GlobalQueueInner();
GlobalQueue.SetQueue(_q);
var connection = new ConnectionConfiguration("testService", 10);
var distrconfig = new DistributorHashConfiguration(countReplics);
var queueconfig = new QueueConfiguration(1, 100);
_dnet = new DistributorNetModule(connection,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
Distributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
queueconfig, _dnet,
new ServerId("localhost", distrServer1),
new ServerId("localhost", distrServer12),
new HashMapConfiguration(hashFile,
HashMapCreationMode.ReadFromFile,
1, countReplics, HashFileType.Distributor));
_dnet.SetDistributor(Distributor);
_tranc = new TransactionModule(new QueueConfiguration(1, 1000), _dnet, new TransactionConfiguration(4),
distrconfig);
Main =
new MainLogicModule(new DistributorTimeoutCache(TimeSpan.FromSeconds(200), TimeSpan.FromSeconds(200)),
Distributor, _tranc);
var netReceive1 = new NetReceiverConfiguration(distrServer1, "localhost", "testService");
var netReceive2 = new NetReceiverConfiguration(distrServer12, "localhost", "testService");
Input = new InputModuleWithParallel(new QueueConfiguration(2, 100000), Main, _tranc);
_receiver = new NetDistributorReceiver(Main, Input, Distributor, netReceive1, netReceive2);
}
public void Build(int storageServer, string hashFile, int countReplics)
{
Q = new GlobalQueueInner();
GlobalQueue.SetQueue(Q);
var queueConfiguration = new QueueConfiguration(1, 1000);
var hashMapConfiguration = new HashMapConfiguration(hashFile,
HashMapCreationMode.ReadFromFile, 1, countReplics, HashFileType.Writer);
var local = new ServerId("localhost", storageServer);
_net = new WriterNetModule(new ConnectionConfiguration("testService", 10),
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
Db = new DbModuleCollection();
Db.AddDbModule(new TestDbInMemory());
_async = new AsyncTaskModule(new QueueConfiguration(1, 10));
Restore = new AsyncDbWorkModule(_net, _async, Db,
new RestoreModuleConfiguration(10, TimeSpan.FromMinutes(100)),
new RestoreModuleConfiguration(10, TimeSpan.FromMilliseconds(100)),
new RestoreModuleConfiguration(-1, TimeSpan.FromHours(1), false, TimeSpan.FromHours(1)),
new QueueConfiguration(1, 100), local);
Distributor = new DistributorModule(_async, Restore, _net, local,
hashMapConfiguration, new QueueConfiguration(2, 10), Db);
_mainС = new MainLogicModule(Distributor, Db);
Input = new InputModule(_mainС, queueConfiguration);
_netRc = new NetWriterReceiver(Input, Distributor,
new NetReceiverConfiguration(storageServer, "localhost", "testService"),
new NetReceiverConfiguration(1, "fake", "fake"));
}
public NetDistributorReceiverForDb(DistributorModule distributorModule,
NetReceiverConfiguration receiverConfiguration)
: base(receiverConfiguration)
{
Contract.Requires(distributorModule != null);
_distributorModule = distributorModule;
}
public override void Build()
{
var async = new AsyncTaskModule(new QueueConfiguration(4, 10));
var distributor = new DistributorModule(
new CollectorModel(_distributorHashConfiguration, _hashMapConfiguration, _useHashFile), async,
new AsyncTasksConfiguration(TimeSpan.FromSeconds(10)));
var net = new CollectorNetModule(_connectionConfiguration, _connectionTimeoutConfiguration, distributor);
distributor.SetNetModule(net);
var back = new BackgroundModule(_queueConfiguration);
var loader = new DataLoader(net, _serverPageSize, back);
var searchModule = new SearchTaskCommonModule(loader, distributor, back);
CreateApi = searchModule.CreateApi;
Distributor = distributor;
AddModule(back);
AddModule(net);
AddModule(loader);
AddModule(distributor);
AddModule(searchModule);
AddModule(async);
AddModuleDispose(async);
AddModuleDispose(searchModule);
AddModuleDispose(loader);
AddModuleDispose(back);
AddModuleDispose(net);
AddModuleDispose(distributor);
}
public NetDistributorReceiver(MainLogicModule main, IInputModule input, DistributorModule distributorModule,
NetReceiverConfiguration receiverConfigurationForDb,
NetReceiverConfiguration receiverConfigurationForFroxy)
{
_distributorReceiverForDb = new NetDistributorReceiverForDb(distributorModule, receiverConfigurationForDb);
_distributorReceiverForProxy = new NetDistributorReceiverForProxy(main, input, distributorModule,
receiverConfigurationForFroxy);
}
public NetWriterReceiverForWrite(InputModule inputModule, DistributorModule distributor, NetReceiverConfiguration receiverConfiguration)
: base(receiverConfiguration)
{
Contract.Requires(inputModule != null);
Contract.Requires(distributor != null);
_distributor = distributor;
_inputModule = inputModule;
}
public SearchTaskCommonModule(IDataLoader dataLoader, DistributorModule distributor,
BackgroundModule backgroundModule)
{
_dataLoader = dataLoader;
_distributor = distributor;
_backgroundModule = backgroundModule;
_apis = new Dictionary <string, SearchTaskModule>();
}
public void MainLogicModule_TransactionAnswerResult_ReceiveAnswersFromWriter()
{
const int distrServer1 = 22168;
const int distrServer2 = 23168;
#region hell
var connectionConf = new ConnectionConfiguration("testService", 10);
var distrconfig = new DistributorHashConfiguration(1);
var queueconfig = new QueueConfiguration(1, 100);
var dnet = new DistributorNetModule(connectionConf,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var ddistributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
queueconfig, dnet,
new ServerId("localhost", distrServer1),
new ServerId("localhost", distrServer2),
new HashMapConfiguration("TestDistributorReceiveAndDbSendAsync",
HashMapCreationMode.ReadFromFile,
1, 1, HashFileType.Distributor));
dnet.SetDistributor(ddistributor);
var tranc = new TransactionModule(new QueueConfiguration(1, 1000), dnet, new TransactionConfiguration(4),
distrconfig);
var cache = new DistributorTimeoutCache(TimeSpan.FromMilliseconds(2000), TimeSpan.FromMilliseconds(200000));
var main = new MainLogicModule(cache, ddistributor, tranc);
cache.SetMainLogicModule(main);
var netReceive4 = new NetReceiverConfiguration(distrServer1, "localhost", "testService");
var netReceive42 = new NetReceiverConfiguration(distrServer2, "localhost", "testService");
var input = new InputModuleWithParallel(new QueueConfiguration(2, 100000), main, tranc);
var receiver4 = new NetDistributorReceiver(main, input,
ddistributor, netReceive4, netReceive42);
ddistributor.Start();
receiver4.Start();
#endregion
var t = 0;
GlobalQueue.Queue.TransactionQueue.Registrate(data => Interlocked.Increment(ref t));
GlobalQueue.Queue.Start();
var connection = new SingleConnectionToDistributor(
new ServerId("localhost", distrServer1), new ConnectionConfiguration("testService", 10),
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
connection.Connect();
connection.TransactionAnswerResult(new Transaction("123", "123"));
Thread.Sleep(TimeSpan.FromMilliseconds(100));
connection.TransactionAnswerResult(new Transaction("1243", "1423"));
Thread.Sleep(TimeSpan.FromMilliseconds(100));
Assert.AreEqual(2, t);
connection.Dispose();
receiver4.Dispose();
}
public void TransactionModule_ProcessSyncWithExecutor_RollbackNoEnoughServers()
{
var server1 = new ServerId("localhost", 21141);
var server2 = new ServerId("localhost", 21142);
var server3 = new ServerId("localhost", 21143);
var netconfig = new ConnectionConfiguration("testService", 10);
var queueconfig = new QueueConfiguration(1, 100);
var distrconfig = new DistributorHashConfiguration(2);
var distributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
queueconfig, null, new ServerId("localhost", 1),
new ServerId("localhost", 1),
new HashMapConfiguration("test10", HashMapCreationMode.CreateNew, 1, 1, HashFileType.Distributor));
var net = new DistributorNetModule(netconfig,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
net.SetDistributor(distributor);
distributor.Start();
net.Start();
GlobalQueue.Queue.Start();
var s1 = TestHelper.OpenWriterHost(server1, netconfig);
var s2 = TestHelper.OpenWriterHost(server2, netconfig);
net.ConnectToWriter(server1);
net.ConnectToWriter(server2);
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
var ev = new InnerData(new Transaction("", ""))
{
Transaction = { Destination = new List <ServerId> {
server1, server2, server3
} }
};
var trm = new TransactionModule(new QueueConfiguration(1, 1000), net, new TransactionConfiguration(1),
new DistributorHashConfiguration(3));
trm.Start();
using (var trans = trm.Rent())
{
trm.ProcessSyncWithExecutor(ev, trans.Element);
}
Thread.Sleep(TimeSpan.FromMilliseconds(100));
Assert.IsTrue(s1.Value <= 0);
Assert.IsTrue(s2.Value <= 0);
Assert.IsTrue(ev.Transaction.IsError);
net.Dispose();
trm.Dispose();
}
public MainLogicModule(DistributorModule distributor, DbModule db)
{
Contract.Requires(distributor != null);
Contract.Requires(db != null);
_db = db;
_distributor = distributor;
_queue = GlobalQueue.Queue;
}
public NetWriterReceiver(InputModule inputModule, DistributorModule distributor,
NetReceiverConfiguration receiverConfigurationForWrite,
NetReceiverConfiguration receiverConfigurationForCollector)
{
_writerReceiverForWrite = new NetWriterReceiverForWrite(inputModule, distributor,
receiverConfigurationForWrite);
_writerReceiverForCollector = new NetWriterReceiverForCollector(inputModule, distributor,
receiverConfigurationForCollector);
}
public NetDistributorReceiverForProxy(MainLogicModule main, IInputModule input, DistributorModule distributorModule,
NetReceiverConfiguration receiverConfiguration) : base(receiverConfiguration)
{
Contract.Requires(main != null);
Contract.Requires(input != null);
Contract.Requires(distributorModule != null);
_mainLogic = main;
_input = input;
_distributorModule = distributorModule;
}
public SearchTaskModule(string tableName, MergeBase mergeBase, IDataLoader dataLoader, DistributorModule distributor,
BackgroundModule backgroundModule, ScriptParser scriptParser)
{
_tableName = tableName;
_mergeBase = mergeBase;
_dataLoader = dataLoader;
_distributor = distributor;
_backgroundModule = backgroundModule;
_scriptParser = scriptParser;
}
public void NetModule_Process_SendDatatoAvaliableAndUnavalilableServers()
{
var server1 = new ServerId("localhost", 21121);
var server2 = new ServerId("localhost", 21122);
var server3 = new ServerId("localhost", 21123);
var netconfig = new ConnectionConfiguration("testService", 10);
var queueconfig = new QueueConfiguration(1, 100);
var distrconfig = new DistributorHashConfiguration(2);
var s1 = TestHelper.OpenWriterHost(server1, netconfig);
var s2 = TestHelper.OpenWriterHost(server2, netconfig);
var net = new DistributorNetModule(netconfig,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var distributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
queueconfig, net, new ServerId("localhost", 1),
new ServerId("localhost", 1),
new HashMapConfiguration("test12", HashMapCreationMode.CreateNew, 1,
1, HashFileType.Distributor));
net.SetDistributor(distributor);
distributor.Start();
net.Start();
GlobalQueue.Queue.Start();
net.ConnectToWriter(server1);
net.ConnectToWriter(server2);
var ev = new InnerData(new Transaction("", ""))
{
Transaction = { Destination = new List <ServerId> {
server1
} }
};
var ret1 = net.Process(server1, ev);
var ret2 = net.Process(server2, ev);
var ret3 = net.Process(server3, ev);
Assert.AreEqual(1, s1.Value);
Assert.AreEqual(1, s2.Value);
Assert.AreEqual(typeof(SuccessResult), ret1.GetType());
Assert.AreEqual(typeof(SuccessResult), ret2.GetType());
Assert.AreEqual(typeof(ServerNotFoundResult), ret3.GetType());
GlobalQueue.Queue.Dispose();
net.Dispose();
}
public override void Build()
{
var q = new GlobalQueueInner();
GlobalQueue.SetQueue(q);
var db = new DbModuleCollection();
var net = new WriterNetModule(_connectionConfiguration, _connectionTimeoutConfiguration);
var async = new AsyncTaskModule(_queueConfiguration);
var restore = new AsyncDbWorkModule(net, async, db, _initiatorRestoreConfiguration,
_transferRestoreConfiguration, _timeoutRestoreConfiguration,
_queueConfigurationRestore, _local, _isNeedRestore);
var distributor = new DistributorModule(async, restore, net, _local, _hashMapConfiguration,
_queueConfiguration, db);
Distributor = distributor;
DbModule = db;
var main = new MainLogicModule(distributor, db);
var input = new InputModule(main, _queueConfiguration);
var receiver = new NetWriterReceiver(input, distributor, _receiverConfigurationForWrite,
_receiverConfigurationForCollector);
AddModule(distributor);
AddModule(input);
AddModule(db);
AddModule(async);
AddModule(restore);
AddModule(main);
AddModule(receiver);
AddModule(q);
AddModuleDispose(receiver);
AddModuleDispose(restore);
AddModuleDispose(async);
AddModuleDispose(q);
AddModuleDispose(input);
AddModuleDispose(main);
AddModuleDispose(distributor);
AddModuleDispose(db);
AddModuleDispose(net);
}
public override void Build()
{
var q = new GlobalQueueInner();
GlobalQueue.SetQueue(q);
var cache = new DistributorTimeoutCache(_cacheConfiguration.TimeAliveBeforeDeleteMls,
_cacheConfiguration.TimeAliveAfterUpdateMls);
var net = CreateNetModule(_connectionConfiguration);
var distributor = new DistributorModule(_pingConfig, _checkConfig, _distributorHashConfiguration,
new QueueConfiguration(1, 1000), net,
_localfordb, _localforproxy, _hashMapConfiguration);
Distributor = distributor;
net.SetDistributor(distributor);
var transaction = new TransactionModule(_queueConfiguration, net, _transactionConfiguration,
_distributorHashConfiguration);
var main = new MainLogicModule(cache, distributor, transaction);
cache.SetMainLogicModule(main);
var input = new InputModuleWithParallel(_queueConfiguration, main, transaction);
var receive = new NetDistributorReceiver(main, input, distributor, _receiverConfigurationForDb,
_receiverConfigurationForProxy);
AddModule(receive);
AddModule(transaction);
AddModule(input);
AddModule(main);
AddModule(net);
AddModule(distributor);
AddModule(q);
AddModuleDispose(receive);
AddModuleDispose(q);
AddModuleDispose(input);
AddModuleDispose(cache);
AddModuleDispose(main);
AddModuleDispose(distributor);
AddModuleDispose(transaction);
AddModuleDispose(net);
}
public void DistributorTimeoutCache_TimeoutData_SendToMainLogicModuleObsoleteData()
{
var cache = new DistributorTimeoutCache(TimeSpan.FromMilliseconds(200), TimeSpan.FromMilliseconds(500));
var net = new DistributorNetModule(new ConnectionConfiguration("", 1),
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var distributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new DistributorHashConfiguration(1), new QueueConfiguration(1, 1),
net, new ServerId("", -1), new ServerId("", -1),
new HashMapConfiguration("", HashMapCreationMode.CreateNew, 1, 1, HashFileType.Distributor));
var trans = new TransactionModule(new QueueConfiguration(1, 1000), net, new TransactionConfiguration(1),
new DistributorHashConfiguration(1));
var main = new MainLogicModule(cache, distributor, trans);
cache.SetMainLogicModule(main);
var ev = new InnerData(new Transaction("123", "")
{
OperationName = OperationName.Create
})
{
Transaction = { Destination = new List <ServerId>() }
};
ev.Transaction.Complete();
cache.AddToCache(ev.Transaction.CacheKey, ev.Transaction);
var ret = cache.Get(ev.Transaction.CacheKey);
Assert.AreEqual(ev.Transaction, ret);
Thread.Sleep(200);
cache.Get(ev.Transaction.CacheKey);
Assert.AreEqual(TransactionState.Complete, ev.Transaction.State);
Thread.Sleep(200);
cache.Get(ev.Transaction.CacheKey);
Assert.AreEqual(TransactionState.Complete, ev.Transaction.State);
Thread.Sleep(500);
ret = cache.Get(ev.Transaction.CacheKey);
Assert.AreEqual(null, ret);
ev = new InnerData(new Transaction("1231", "")
{
OperationName = OperationName.Create
})
{
Transaction = { Destination = new List <ServerId>() }
};
ev.Transaction.StartTransaction();
cache.AddToCache(ev.Transaction.CacheKey, ev.Transaction);
ret = cache.Get(ev.Transaction.CacheKey);
Assert.AreEqual(ev.Transaction, ret);
Thread.Sleep(200);
cache.Get(ev.Transaction.CacheKey);
Assert.AreEqual(TransactionState.Error, ev.Transaction.State);
Thread.Sleep(200);
cache.Get(ev.Transaction.CacheKey);
Assert.AreEqual(TransactionState.Error, ev.Transaction.State);
Thread.Sleep(500);
ret = cache.Get(ev.Transaction.CacheKey);
Assert.AreEqual(null, ret);
}
public void CollectorNet_ReadFromWriter()
{
const int proxyServer = 22337;
const int distrServer1 = 22338;
const int distrServer12 = 22339;
const int st1 = 22335;
const int st2 = 22336;
#region hell
var writer = new HashWriter(new HashMapConfiguration("TestCollectorNet", HashMapCreationMode.CreateNew, 1, 1, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", st1, st2);
writer.Save();
var common = new CommonConfiguration(1, 100);
var netconfig = new NetConfiguration("localhost", proxyServer, "testService", 10);
var toconfig = new ProxyConfiguration(TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(1),
TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(10));
var proxy = new TestGate(netconfig, toconfig, common);
var distrNet = new DistributorNetConfiguration("localhost",
distrServer1, distrServer12, "testService", 10);
var distrConf = new DistributorConfiguration(1, "TestCollectorNet",
TimeSpan.FromMilliseconds(100000), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), TimeSpan.FromMilliseconds(10000));
var distr = new DistributorApi(distrNet, distrConf, common);
var storageNet = new StorageNetConfiguration("localhost", st1, st2, "testService", 10);
var storageConfig = new StorageConfiguration("TestCollectorNet", 1, 10, TimeSpan.FromHours(1), TimeSpan.FromHours(1),
TimeSpan.FromHours(1), TimeSpan.FromHours(1), false);
var storage = new WriterApi(storageNet, storageConfig, common);
var async = new AsyncTaskModule(new QueueConfiguration(4, 10));
var distributor =
new DistributorModule(new CollectorModel(new DistributorHashConfiguration(1),
new HashMapConfiguration("TestCollectorNet", HashMapCreationMode.ReadFromFile, 1, 1,
HashFileType.Collector)), async, new AsyncTasksConfiguration(TimeSpan.FromMinutes(1)));
var net = new CollectorNetModule(new ConnectionConfiguration("testService", 10),
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout), distributor);
distributor.SetNetModule(net);
var back = new BackgroundModule(new QueueConfiguration(5, 10));
var loader = new DataLoader(net, 100, back);
var parser = new TestIntParser();
parser.SetCommandsHandler(
new UserCommandsHandler <TestCommand, Type, TestCommand, int, int, TestDbReader>(
new TestUserCommandCreator(), new TestMetaDataCommandCreator()));
var merge = new OrderMerge(loader, parser);
var searchModule = new SearchTaskModule("Int", merge, loader, distributor, back, parser);
storage.Build();
proxy.Build();
distr.Build();
storage.AddDbModule(new TestInMemoryDbFactory());
storage.Start();
proxy.Start();
distr.Start();
searchModule.Start();
distributor.Start();
merge.Start();
back.Start();
net.Start();
async.Start();
#endregion
proxy.Int.SayIAmHere("localhost", distrServer1);
const int count = 20;
for (int i = 0; i < count; i++)
{
var request = proxy.Int.CreateSync(i, i);
Assert.AreEqual(RequestState.Complete, request.State);
}
var reader = searchModule.CreateReader("asc", -1, 20);
reader.Start();
for (int i = 0; i < count; i++)
{
Assert.IsTrue(reader.IsCanRead);
reader.ReadNext();
Assert.AreEqual(i, reader.GetValue(0));
}
reader.ReadNext();
Assert.IsFalse(reader.IsCanRead);
reader.Dispose();
back.Dispose();
net.Dispose();
storage.Dispose();
proxy.Dispose();
distr.Dispose();
async.Dispose();
}
public void SearchTaskModule_CreateReader_UnlimitDataReadAndUserPage()
{
var server1 = new ServerId("", 1);
var server2 = new ServerId("", 2);
var server3 = new ServerId("", 3);
const int pageSize = 5;
var writer = new HashWriter(new HashMapConfiguration("TestCollector4", HashMapCreationMode.CreateNew, 3, 3, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, server1.RemoteHost, server1.Port, 157);
writer.SetServer(1, server2.RemoteHost, server2.Port, 157);
writer.SetServer(2, server3.RemoteHost, server3.Port, 157);
writer.Save();
var loader = new TestDataLoader(pageSize);
var parser = new TestIntParser();
parser.SetCommandsHandler(
new UserCommandsHandler <TestCommand, Type, TestCommand, int, int, TestDbReader>(
new TestUserCommandCreator(), new TestMetaDataCommandCreator()));
var merge = new OrderMerge(loader, parser);
var async = new AsyncTaskModule(new QueueConfiguration(4, 10));
var distributor =
new DistributorModule(new CollectorModel(new DistributorHashConfiguration(1),
new HashMapConfiguration("TestCollector4", HashMapCreationMode.ReadFromFile, 1, 1,
HashFileType.Writer)), async, new AsyncTasksConfiguration(TimeSpan.FromMinutes(1)));
var back = new BackgroundModule(new QueueConfiguration(5, 10));
var searchModule = new SearchTaskModule("", merge, loader, distributor, back, parser);
#region hell
loader.Data.Add(server1, new List <SearchData>
{
TestHelper.CreateData(1),
TestHelper.CreateData(2),
TestHelper.CreateData(4),
TestHelper.CreateData(5),
TestHelper.CreateData(6),
TestHelper.CreateData(7),
TestHelper.CreateData(8),
});
loader.Data.Add(server2, new List <SearchData>
{
TestHelper.CreateData(4),
TestHelper.CreateData(5),
TestHelper.CreateData(6),
TestHelper.CreateData(7),
TestHelper.CreateData(8),
TestHelper.CreateData(9),
TestHelper.CreateData(10),
TestHelper.CreateData(11),
});
loader.Data.Add(server3, new List <SearchData>
{
TestHelper.CreateData(2),
TestHelper.CreateData(3),
TestHelper.CreateData(5),
TestHelper.CreateData(7),
TestHelper.CreateData(8),
TestHelper.CreateData(9),
TestHelper.CreateData(10),
TestHelper.CreateData(11),
TestHelper.CreateData(12),
TestHelper.CreateData(13),
});
#endregion
searchModule.Start();
distributor.Start();
merge.Start();
back.Start();
async.Start();
var reader = searchModule.CreateReader("asc", -1, 5);
reader.Start();
const int count = 13;
for (int i = 0; i < count; i++)
{
Assert.IsTrue(reader.IsCanRead);
reader.ReadNext();
Assert.AreEqual(i + 1, reader.GetValue(0));
}
reader.ReadNext();
Assert.IsFalse(reader.IsCanRead);
reader.Dispose();
async.Dispose();
back.Dispose();
}
public void MainLogic_ProcessWithData_SendAllReplicsThenObsoleteDataInCache()
{
var writer = new HashWriter(new HashMapConfiguration("test9", HashMapCreationMode.CreateNew, 2, 3, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", 21111, 157);
writer.SetServer(1, "localhost", 21112, 157);
writer.Save();
var cache = new DistributorTimeoutCache(TimeSpan.FromMilliseconds(400), TimeSpan.FromMilliseconds(1000));
var distrconfig = new DistributorHashConfiguration(2);
var queueconfig = new QueueConfiguration(1, 100);
var netconfig = new ConnectionConfiguration("testService", 10);
var net = new DistributorNetModule(netconfig,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var distributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
queueconfig, net, new ServerId("localhost", 1),
new ServerId("localhost", 1),
new HashMapConfiguration("test9", HashMapCreationMode.ReadFromFile,
1, 1, HashFileType.Distributor));
net.SetDistributor(distributor);
var transaction = new TransactionModule(new QueueConfiguration(1, 1000), net, new TransactionConfiguration(1),
distrconfig);
var main = new MainLogicModule(cache, distributor, transaction);
cache.SetMainLogicModule(main);
var server1 = new ServerId("localhost", 21111);
var server2 = new ServerId("localhost", 21112);
var s1 = TestHelper.OpenWriterHost(server1, netconfig);
var s2 = TestHelper.OpenWriterHost(server2, netconfig);
cache.Start();
distributor.Start();
net.Start();
transaction.Start();
main.Start();
GlobalQueue.Queue.Start();
net.ConnectToWriter(server1);
net.ConnectToWriter(server2);
Thread.Sleep(TimeSpan.FromMilliseconds(300));
var ev = new InnerData(new Transaction("123", ""))
{
Transaction =
new Transaction(HashConvertor.GetString("1"), "")
{
OperationName = OperationName.Create,
OperationType = OperationType.Async
}
};
ev.Transaction.Destination = new List <ServerId>();
using (var trans = transaction.Rent())
{
main.ProcessWithData(ev, trans.Element);
}
GlobalQueue.Queue.TransactionQueue.Add(ev.Transaction);
GlobalQueue.Queue.TransactionQueue.Add(ev.Transaction);
Thread.Sleep(TimeSpan.FromMilliseconds(100));
Assert.IsTrue(s1.Value > 0);
Assert.IsTrue(s2.Value > 0);
Assert.AreEqual(main.GetTransactionState(ev.Transaction.UserTransaction).State, TransactionState.Complete);
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
Assert.AreEqual(main.GetTransactionState(ev.Transaction.UserTransaction).State, TransactionState.DontExist);
net.Dispose();
distributor.Dispose();
transaction.Dispose();
}
public void InputModuleWithParallel_ProcessAsync_SendToOneServersAndTimeoutInCache_Success()
{
const int distrServer1 = 22166;
const int distrServer2 = 23166;
const int storageServer1 = 22167;
var q1 = new GlobalQueueInner();
var writer =
new HashWriter(new HashMapConfiguration("testAsyncTrans1S", HashMapCreationMode.CreateNew, 1, 1, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", storageServer1, 157);
writer.Save();
#region hell
GlobalQueue.SetQueue(q1);
var connection = new ConnectionConfiguration("testService", 10);
var distrconfig = new DistributorHashConfiguration(1);
var queueconfig = new QueueConfiguration(1, 100);
var dnet = new DistributorNetModule(connection,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var ddistributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
queueconfig, dnet,
new ServerId("localhost", distrServer1),
new ServerId("localhost", distrServer2),
new HashMapConfiguration("testAsyncTrans1S",
HashMapCreationMode.ReadFromFile,
1, 1, HashFileType.Distributor));
dnet.SetDistributor(ddistributor);
var tranc = new TransactionModule(new QueueConfiguration(1, 1000), dnet, new TransactionConfiguration(4),
distrconfig);
var cache = new DistributorTimeoutCache(TimeSpan.FromMilliseconds(2000), TimeSpan.FromMilliseconds(200000));
var main = new MainLogicModule(cache, ddistributor, tranc);
cache.SetMainLogicModule(main);
var netReceive4 = new NetReceiverConfiguration(distrServer1, "localhost", "testService");
var netReceive42 = new NetReceiverConfiguration(distrServer2, "localhost", "testService");
var input = new InputModuleWithParallel(new QueueConfiguration(2, 100000), main, tranc);
var receiver4 = new NetDistributorReceiver(main, input, ddistributor, netReceive4, netReceive42);
#endregion
var s = TestHelper.OpenWriterHost(new ServerId("localhost", storageServer1),
new ConnectionConfiguration("testService", 10));
main.Start();
receiver4.Start();
input.Start();
dnet.Start();
ddistributor.Start();
q1.Start();
var list = new List <InnerData>();
const int count = 1;
for (int i = 1; i < count + 1; i++)
{
var ev =
new InnerData(new Transaction(HashConvertor.GetString(i.ToString(CultureInfo.InvariantCulture)), "")
{
OperationName = OperationName.Create,
OperationType = OperationType.Async
})
{
Data = CommonDataSerializer.Serialize(i)
};
list.Add(ev);
}
foreach (var data in list)
{
input.ProcessAsync(data);
}
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
Assert.AreEqual(count, s.Value);
foreach (var data in list)
{
var transaction = main.GetTransactionState(data.Transaction.UserTransaction);
Assert.AreEqual(TransactionState.TransactionInProcess, transaction.State);
}
Thread.Sleep(TimeSpan.FromMilliseconds(2000));
foreach (var data in list)
{
var transaction = main.GetTransactionState(data.Transaction.UserTransaction);
if (transaction.State == TransactionState.DontExist)
{
Thread.Sleep(1000);
}
transaction = main.GetTransactionState(data.Transaction.UserTransaction);
Assert.AreEqual(TransactionState.Error, transaction.State);
}
q1.Dispose();
ddistributor.Dispose();
dnet.Dispose();
cache.Dispose();
}
public void AsyncTaskModule_PingServers_AvalilableAfterSomeTime()
{
const int storageServer1 = 22131;
const int storageServer2 = 22132;
const int distrServer1 = 22134;
const int distrServer12 = 23134;
var writer = new HashWriter(new HashMapConfiguration("TestAsyncPing", HashMapCreationMode.CreateNew, 2, 3, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", storageServer1, 157);
writer.SetServer(1, "localhost", storageServer2, 157);
writer.Save();
#region hell
var connection = new ConnectionConfiguration("testService", 10);
var distrconfig = new DistributorHashConfiguration(1);
var queueconfig = new QueueConfiguration(1, 100);
var dnet = new DistributorNetModule(connection,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var ddistributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMilliseconds(200)),
new AsyncTasksConfiguration(TimeSpan.FromMilliseconds(200)),
distrconfig,
queueconfig, dnet,
new ServerId("localhost", distrServer1),
new ServerId("localhost", distrServer12),
new HashMapConfiguration("TestAsyncPing", HashMapCreationMode.ReadFromFile, 1, 1, HashFileType.Distributor));
dnet.SetDistributor(ddistributor);
dnet.Start();
ddistributor.Start();
GlobalQueue.Queue.Start();
#endregion
var data1 = new InnerData(new Transaction("", ""));
var data2 = new InnerData(new Transaction("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", ""));
var dest = ddistributor.GetDestination(data1, false);
var dest2 = ddistributor.GetDestination(data2, false);
dnet.Process(dest.First(), data1);
dnet.Process(dest2.First(), data1);
Thread.Sleep(100);
dest = ddistributor.GetDestination(data1, false);
dest2 = ddistributor.GetDestination(data2, false);
Assert.AreEqual(null, dest);
Assert.AreEqual(null, dest2);
TestHelper.OpenWriterHost(new ServerId("localhost", storageServer1),
new ConnectionConfiguration("testService", 10));
TestHelper.OpenWriterHost(new ServerId("localhost", storageServer2),
new ConnectionConfiguration("testService", 10));
Thread.Sleep(TimeSpan.FromMilliseconds(800));
Assert.AreEqual(1, ddistributor.GetDestination(data1, false).Count);
Assert.AreEqual(1, ddistributor.GetDestination(data2, false).Count);
ddistributor.Dispose();
}
public CollectorNetModule(ConnectionConfiguration connectionConfiguration,
ConnectionTimeoutConfiguration connectionTimeout, DistributorModule distributor)
: base(connectionConfiguration, connectionTimeout)
{
_distributor = distributor;
}
public void SetDistributor(DistributorModule distributor)
{
Contract.Requires(distributor != null);
_distributor = distributor;
}
public void ProxyDistributorModule_SayIAmHere_AddDistributor()
{
const int server1 = 22250;
const int server2 = 22251;
const int server3 = 22252;
const int server4 = 22253;
const int server42 = 23253;
const int server5 = 22254;
const int server52 = 23254;
var q1 = new GlobalQueueInner();
var q2 = new GlobalQueueInner();
var q3 = new GlobalQueueInner();
GlobalQueue.SetQueue(q1);
var queue = new QueueConfiguration(1, 1000);
var netconfig = new ConnectionConfiguration("testService", 10);
var netReceive = new NetReceiverConfiguration(server1, "localhost", "testService");
var net = new ProxyNetModule(netconfig,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var distributor = new ProxyDistributorModule(new AsyncProxyCache(TimeSpan.FromMinutes(100)), net, queue, new ServerId("localhost", server1),
new AsyncTasksConfiguration(TimeSpan.FromDays(1)),
new AsyncTasksConfiguration(TimeSpan.FromDays(1)));
net.SetDistributor(distributor);
var receive = new ProxyNetReceiver(distributor, netReceive);
GlobalQueue.SetQueue(q2);
var netReceive2 = new NetReceiverConfiguration(server2, "localhost", "testService");
var net2 = new ProxyNetModule(netconfig,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var distributor2 = new ProxyDistributorModule(new AsyncProxyCache(TimeSpan.FromMinutes(100)), net2, queue, new ServerId("localhost", server2),
new AsyncTasksConfiguration(TimeSpan.FromDays(1)),
new AsyncTasksConfiguration(TimeSpan.FromDays(1)));
net2.SetDistributor(distributor2);
var receive2 = new ProxyNetReceiver(distributor2, netReceive2);
GlobalQueue.SetQueue(q3);
var netReceive3 = new NetReceiverConfiguration(server3, "localhost", "testService");
var net3 = new ProxyNetModule(netconfig,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var distributor3 = new ProxyDistributorModule(new AsyncProxyCache(TimeSpan.FromMinutes(100)), net3, queue, new ServerId("localhost", server3),
new AsyncTasksConfiguration(TimeSpan.FromDays(1)),
new AsyncTasksConfiguration(TimeSpan.FromDays(1)));
net3.SetDistributor(distributor3);
var receive3 = new ProxyNetReceiver(distributor3, netReceive3);
var distrconfig = new DistributorHashConfiguration(2);
var queueconfig = new QueueConfiguration(1, 100);
var dnet = new DistributorNetModule(netconfig,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var ddistributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
queueconfig, dnet, new ServerId("localhost", server4),
new ServerId("localhost", server42),
new HashMapConfiguration("test7", HashMapCreationMode.CreateNew, 1, 1, HashFileType.Distributor));
dnet.SetDistributor(ddistributor);
var tranc = new TransactionModule(new QueueConfiguration(1, 1000), dnet, new TransactionConfiguration(4),
new DistributorHashConfiguration(1));
var main = new MainLogicModule(new DistributorTimeoutCache(new TimeSpan(), new TimeSpan()), ddistributor,
tranc);
var netReceive4 = new NetReceiverConfiguration(server4, "localhost", "testService");
var netReceive42 = new NetReceiverConfiguration(server42, "localhost", "testService");
var input = new InputModuleWithParallel(new QueueConfiguration(1, 1), main, tranc);
var receiver4 = new NetDistributorReceiver(main, input, ddistributor, netReceive4, netReceive42);
GlobalQueue.SetQueue(q1);
var dnet2 = new DistributorNetModule(netconfig,
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
var ddistributor2 = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
queueconfig, dnet2,
new ServerId("localhost", server5),
new ServerId("localhost", server52),
new HashMapConfiguration("test6", HashMapCreationMode.CreateNew, 1, 1, HashFileType.Distributor));
dnet2.SetDistributor(ddistributor2);
var netReceive5 = new NetReceiverConfiguration(server5, "localhost", "testService");
var netReceive52 = new NetReceiverConfiguration(server52, "localhost", "testService");
var receiver5 = new NetDistributorReceiver(main,
new InputModuleWithParallel(new QueueConfiguration(1, 1), main,
tranc),
ddistributor2, netReceive5, netReceive52);
receive.Start();
receive2.Start();
receive3.Start();
receiver4.Start();
receiver5.Start();
net.Start();
net2.Start();
net3.Start();
distributor.Start();
distributor2.Start();
distributor3.Start();
dnet.Start();
dnet2.Start();
ddistributor.Start();
ddistributor2.Start();
q1.Start();
q2.Start();
q3.Start();
distributor.SayIAmHere(new ServerId("localhost", server42));
distributor2.SayIAmHere(new ServerId("localhost", server42));
var privateObject = new PrivateObject(distributor);
var dsm1 = (DistributorSystemModel)privateObject.GetField("_distributorSystemModel");
privateObject = new PrivateObject(distributor2);
var dsm2 = (DistributorSystemModel)privateObject.GetField("_distributorSystemModel");
Assert.AreEqual(1, dsm1.GetDistributorsList().Count);
Assert.AreEqual(1, dsm2.GetDistributorsList().Count);
ddistributor2.SayIAmHereRemoteResult(new ServerId("localhost", server42));
Thread.Sleep(TimeSpan.FromMilliseconds(300));
privateObject = new PrivateObject(ddistributor);
var mad1 =
(Impl.DistributorModules.Model.DistributorSystemModel)
privateObject.GetField("_modelOfAnotherDistributors");
privateObject = new PrivateObject(ddistributor2);
var mad2 =
(Impl.DistributorModules.Model.DistributorSystemModel)
privateObject.GetField("_modelOfAnotherDistributors");
Thread.Sleep(400);
Assert.AreEqual(1, mad1.GetDistributorList().Count);
Assert.AreEqual(1, mad2.GetDistributorList().Count);
distributor3.SayIAmHere(new ServerId("localhost", server52));
privateObject = new PrivateObject(distributor3);
var dsm3 = (DistributorSystemModel)privateObject.GetField("_distributorSystemModel");
Assert.AreEqual(2, dsm3.GetDistributorsList().Count);
q1.Dispose();
q2.Dispose();
q3.Dispose();
net.Dispose();
net2.Dispose();
net3.Dispose();
dnet.Dispose();
dnet2.Dispose();
ddistributor.Dispose();
ddistributor2.Dispose();
distributor.Dispose();
distributor2.Dispose();
distributor3.Dispose();
}
public NetWriterReceiverForCollector(InputModule inputModule, DistributorModule distributor, NetReceiverConfiguration configuration)
: base(configuration)
{
_inputModule = inputModule;
_distributor = distributor;
}
