public override void Build()
{
var async = new AsyncTaskModule(new QueueConfiguration(4, 10));
var distributor = new DistributorModule(
new CollectorModel(_distributorHashConfiguration, _hashMapConfiguration, _useHashFile), async,
new AsyncTasksConfiguration(TimeSpan.FromSeconds(10)));
var net = new CollectorNetModule(_connectionConfiguration, _connectionTimeoutConfiguration, distributor);
distributor.SetNetModule(net);
var back = new BackgroundModule(_queueConfiguration);
var loader = new DataLoader(net, _serverPageSize, back);
var searchModule = new SearchTaskCommonModule(loader, distributor, back);
CreateApi = searchModule.CreateApi;
Distributor = distributor;
AddModule(back);
AddModule(net);
AddModule(loader);
AddModule(distributor);
AddModule(searchModule);
AddModule(async);
AddModuleDispose(async);
AddModuleDispose(searchModule);
AddModuleDispose(loader);
AddModuleDispose(back);
AddModuleDispose(net);
AddModuleDispose(distributor);
}
public DistributorModule(CollectorModel model, AsyncTaskModule asyncTaskModule,
AsyncTasksConfiguration asyncPing)
{
_model = model;
_asyncTaskModule = asyncTaskModule;
_asyncPing = asyncPing;
}
public DistributorModule(
AsyncTasksConfiguration asyncPing,
AsyncTasksConfiguration asyncCheck,
DistributorHashConfiguration configuration,
QueueConfiguration queueConfiguration,
DistributorNetModule distributorNet,
ServerId localfordb,
ServerId localforproxy,
HashMapConfiguration hashMapConfiguration)
{
Contract.Requires(configuration != null);
Contract.Requires(queueConfiguration != null);
Contract.Requires(distributorNet != null);
Contract.Requires(localfordb != null);
Contract.Requires(localforproxy != null);
Contract.Requires(asyncPing != null);
_asyncPing = asyncPing;
_asyncTaskModule = new AsyncTaskModule(queueConfiguration);
_queueConfiguration = queueConfiguration;
_modelOfDbWriters = new WriterSystemModel(configuration, hashMapConfiguration);
_modelOfAnotherDistributors = new DistributorSystemModel();
_distributorNet = distributorNet;
_localfordb = localfordb;
_localforproxy = localforproxy;
_asyncCheck = asyncCheck;
_queue = GlobalQueue.Queue;
}
public DistributorModule(AsyncTaskModule async, AsyncDbWorkModule asyncDbWork,
WriterNetModule writerNet,
ServerId local,
HashMapConfiguration hashMapConfiguration,
QueueConfiguration configuration,
DbModuleCollection dbModuleCollection,
AsyncTasksConfiguration pingConfiguration = null)
{
Contract.Requires(local != null);
Contract.Requires(writerNet != null);
Contract.Requires(configuration != null);
Contract.Requires(asyncDbWork != null);
Contract.Requires(async != null);
Contract.Assert(dbModuleCollection != null);
_async = async;
_asyncDbWork = asyncDbWork;
_model = new WriterModel(local, hashMapConfiguration);
_writerNet = writerNet;
_queueConfiguration = configuration;
_dbModuleCollection = dbModuleCollection;
_queue = GlobalQueue.Queue;
var ping = TimeSpan.FromMinutes(1);
if (pingConfiguration != null)
{
ping = pingConfiguration.TimeoutPeriod;
}
_async.AddAsyncTask(
new AsyncDataPeriod(ping, Ping, AsyncTasksNames.AsyncPing, -1), false);
}
public void Build(int storageServer, string hashFile, int countReplics)
{
Q = new GlobalQueueInner();
GlobalQueue.SetQueue(Q);
var queueConfiguration = new QueueConfiguration(1, 1000);
var hashMapConfiguration = new HashMapConfiguration(hashFile,
HashMapCreationMode.ReadFromFile, 1, countReplics, HashFileType.Writer);
var local = new ServerId("localhost", storageServer);
_net = new WriterNetModule(new ConnectionConfiguration("testService", 10),
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
Db = new DbModuleCollection();
Db.AddDbModule(new TestDbInMemory());
_async = new AsyncTaskModule(new QueueConfiguration(1, 10));
Restore = new AsyncDbWorkModule(_net, _async, Db,
new RestoreModuleConfiguration(10, TimeSpan.FromMinutes(100)),
new RestoreModuleConfiguration(10, TimeSpan.FromMilliseconds(100)),
new RestoreModuleConfiguration(-1, TimeSpan.FromHours(1), false, TimeSpan.FromHours(1)),
new QueueConfiguration(1, 100), local);
Distributor = new DistributorModule(_async, Restore, _net, local,
hashMapConfiguration, new QueueConfiguration(2, 10), Db);
_mainС = new MainLogicModule(Distributor, Db);
Input = new InputModule(_mainС, queueConfiguration);
_netRc = new NetWriterReceiver(Input, Distributor,
new NetReceiverConfiguration(storageServer, "localhost", "testService"),
new NetReceiverConfiguration(1, "fake", "fake"));
}
public InitiatorRestoreModule(RestoreModuleConfiguration configuration, WriterNetModule writerNet,
AsyncTaskModule asyncTaskModule) : base(writerNet, asyncTaskModule)
{
Contract.Requires(configuration != null);
_configuration = configuration;
_lock = new ReaderWriterLockSlim();
}
public CommonAsyncWorkModule(WriterNetModule writerNet, AsyncTaskModule asyncTaskModule)
{
Contract.Requires(writerNet != null);
Contract.Requires(asyncTaskModule != null);
_asyncTaskModule = asyncTaskModule;
WriterNet = writerNet;
_isStart = false;
}
public ProxyDistributorModule(AsyncProxyCache asyncProxyCache, ProxyNetModule net,
QueueConfiguration queueConfiguration, ServerId local,
AsyncTasksConfiguration asyncGetData, AsyncTasksConfiguration asyncPing)
{
_asyncProxyCache = asyncProxyCache;
_asynPing = asyncPing;
_queueConfiguration = queueConfiguration;
_distributorSystemModel = new DistributorSystemModel();
_asynGetData = asyncGetData;
_net = net;
_local = local;
_async = new AsyncTaskModule(queueConfiguration);
_queue = GlobalQueue.Queue;
}
public TimeoutModule(WriterNetModule net, AsyncTaskModule asyncTaskModule,
QueueConfiguration queueConfiguration, DbModuleCollection db,
RestoreModuleConfiguration configuration)
: base(net, asyncTaskModule)
{
_queueConfiguration = queueConfiguration;
_db = db;
_queue = GlobalQueue.Queue.DbTimeoutQueue;
_deleteTimeout = configuration.DeleteTimeout;
_asyncTaskModule.AddAsyncTask(
new AsyncDataPeriod(configuration.PeriodRetry, PeriodMessage,
AsyncTasksNames.TimeoutDelete, -1), configuration.IsForceStart);
}
public TransferRestoreModule(RestoreModuleConfiguration configuration, WriterNetModule writerNet,
AsyncTaskModule asyncTaskModule, DbModuleCollection db, ServerId local,
QueueConfiguration queueConfiguration)
: base(writerNet, asyncTaskModule)
{
Contract.Requires(configuration != null);
Contract.Requires(db != null);
Contract.Requires(local != null);
Contract.Requires(queueConfiguration != null);
_queue = GlobalQueue.Queue;
_db = db;
_configuration = configuration;
_local = local;
_queueConfiguration = queueConfiguration;
}
public void AsyncTaskModule_Dispose_StopAsyncTaskAfterNumberOfRetry()
{
var test = new AsyncTaskModule(new QueueConfiguration(2, -1));
int value = 0;
const string name1 = "test1";
var async1 = new AsyncDataPeriod(TimeSpan.FromMilliseconds(100), async => Interlocked.Increment(ref value),
name1, 4);
test.Start();
test.AddAsyncTask(async1, true);
Thread.Sleep(TimeSpan.FromMilliseconds(800));
Assert.AreEqual(4, value);
test.Dispose();
}
public override void Build()
{
var q = new GlobalQueueInner();
GlobalQueue.SetQueue(q);
var db = new DbModuleCollection();
var net = new WriterNetModule(_connectionConfiguration, _connectionTimeoutConfiguration);
var async = new AsyncTaskModule(_queueConfiguration);
var restore = new AsyncDbWorkModule(net, async, db, _initiatorRestoreConfiguration,
_transferRestoreConfiguration, _timeoutRestoreConfiguration,
_queueConfigurationRestore, _local, _isNeedRestore);
var distributor = new DistributorModule(async, restore, net, _local, _hashMapConfiguration,
_queueConfiguration, db);
Distributor = distributor;
DbModule = db;
var main = new MainLogicModule(distributor, db);
var input = new InputModule(main, _queueConfiguration);
var receiver = new NetWriterReceiver(input, distributor, _receiverConfigurationForWrite,
_receiverConfigurationForCollector);
AddModule(distributor);
AddModule(input);
AddModule(db);
AddModule(async);
AddModule(restore);
AddModule(main);
AddModule(receiver);
AddModule(q);
AddModuleDispose(receiver);
AddModuleDispose(restore);
AddModuleDispose(async);
AddModuleDispose(q);
AddModuleDispose(input);
AddModuleDispose(main);
AddModuleDispose(distributor);
AddModuleDispose(db);
AddModuleDispose(net);
}
public void AsyncTaskModule_AddAsyncTask_AmountOfOperations_2Tasks()
{
var test = new AsyncTaskModule(new QueueConfiguration(2, -1));
int value = 0;
const string name1 = "test1";
const string name2 = "test2";
var async1 = new AsyncDataPeriod(TimeSpan.FromMilliseconds(500), async => Interlocked.Increment(ref value),
name1, -1);
var async2 = new AsyncDataPeriod(TimeSpan.FromMilliseconds(500), async => Interlocked.Increment(ref value),
name2, -1);
test.Start();
test.AddAsyncTask(async1, true);
test.AddAsyncTask(async2, true);
Thread.Sleep(TimeSpan.FromMilliseconds(300));
Assert.AreEqual(2, value);
Thread.Sleep(TimeSpan.FromMilliseconds(500));
Assert.AreEqual(4, value);
test.Dispose();
}
public AsyncDbWorkModule(WriterNetModule writerNet, AsyncTaskModule async, DbModuleCollection db,
RestoreModuleConfiguration initiatorConfiguration,
RestoreModuleConfiguration transferConfiguration,
RestoreModuleConfiguration timeoutConfiguration,
QueueConfiguration queueConfiguration, ServerId local, bool isNeedRestore = false)
{
Contract.Requires(initiatorConfiguration != null);
Contract.Requires(transferConfiguration != null);
Contract.Requires(timeoutConfiguration != null);
Contract.Requires(queueConfiguration != null);
Contract.Requires(db != null);
Contract.Requires(writerNet != null);
Contract.Requires(async != null);
Contract.Requires(local != null);
_stateHelper = new RestoreStateHelper(isNeedRestore);
_initiatorRestore = new InitiatorRestoreModule(initiatorConfiguration, writerNet, async);
_transfer = new TransferRestoreModule(transferConfiguration, writerNet, async,
db, local, queueConfiguration);
_timeout = new TimeoutModule(writerNet, async, queueConfiguration,
db, timeoutConfiguration);
}
public void CollectorNet_ReadFromWriter()
{
const int proxyServer = 22337;
const int distrServer1 = 22338;
const int distrServer12 = 22339;
const int st1 = 22335;
const int st2 = 22336;
#region hell
var writer = new HashWriter(new HashMapConfiguration("TestCollectorNet", HashMapCreationMode.CreateNew, 1, 1, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, "localhost", st1, st2);
writer.Save();
var common = new CommonConfiguration(1, 100);
var netconfig = new NetConfiguration("localhost", proxyServer, "testService", 10);
var toconfig = new ProxyConfiguration(TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(1),
TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(10));
var proxy = new TestGate(netconfig, toconfig, common);
var distrNet = new DistributorNetConfiguration("localhost",
distrServer1, distrServer12, "testService", 10);
var distrConf = new DistributorConfiguration(1, "TestCollectorNet",
TimeSpan.FromMilliseconds(100000), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), TimeSpan.FromMilliseconds(10000));
var distr = new DistributorApi(distrNet, distrConf, common);
var storageNet = new StorageNetConfiguration("localhost", st1, st2, "testService", 10);
var storageConfig = new StorageConfiguration("TestCollectorNet", 1, 10, TimeSpan.FromHours(1), TimeSpan.FromHours(1),
TimeSpan.FromHours(1), TimeSpan.FromHours(1), false);
var storage = new WriterApi(storageNet, storageConfig, common);
var async = new AsyncTaskModule(new QueueConfiguration(4, 10));
var distributor =
new DistributorModule(new CollectorModel(new DistributorHashConfiguration(1),
new HashMapConfiguration("TestCollectorNet", HashMapCreationMode.ReadFromFile, 1, 1,
HashFileType.Collector)), async, new AsyncTasksConfiguration(TimeSpan.FromMinutes(1)));
var net = new CollectorNetModule(new ConnectionConfiguration("testService", 10),
new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout), distributor);
distributor.SetNetModule(net);
var back = new BackgroundModule(new QueueConfiguration(5, 10));
var loader = new DataLoader(net, 100, back);
var parser = new TestIntParser();
parser.SetCommandsHandler(
new UserCommandsHandler <TestCommand, Type, TestCommand, int, int, TestDbReader>(
new TestUserCommandCreator(), new TestMetaDataCommandCreator()));
var merge = new OrderMerge(loader, parser);
var searchModule = new SearchTaskModule("Int", merge, loader, distributor, back, parser);
storage.Build();
proxy.Build();
distr.Build();
storage.AddDbModule(new TestInMemoryDbFactory());
storage.Start();
proxy.Start();
distr.Start();
searchModule.Start();
distributor.Start();
merge.Start();
back.Start();
net.Start();
async.Start();
#endregion
proxy.Int.SayIAmHere("localhost", distrServer1);
const int count = 20;
for (int i = 0; i < count; i++)
{
var request = proxy.Int.CreateSync(i, i);
Assert.AreEqual(RequestState.Complete, request.State);
}
var reader = searchModule.CreateReader("asc", -1, 20);
reader.Start();
for (int i = 0; i < count; i++)
{
Assert.IsTrue(reader.IsCanRead);
reader.ReadNext();
Assert.AreEqual(i, reader.GetValue(0));
}
reader.ReadNext();
Assert.IsFalse(reader.IsCanRead);
reader.Dispose();
back.Dispose();
net.Dispose();
storage.Dispose();
proxy.Dispose();
distr.Dispose();
async.Dispose();
}
public void SearchTaskModule_CreateReader_UnlimitDataReadAndUserPage()
{
var server1 = new ServerId("", 1);
var server2 = new ServerId("", 2);
var server3 = new ServerId("", 3);
const int pageSize = 5;
var writer = new HashWriter(new HashMapConfiguration("TestCollector4", HashMapCreationMode.CreateNew, 3, 3, HashFileType.Distributor));
writer.CreateMap();
writer.SetServer(0, server1.RemoteHost, server1.Port, 157);
writer.SetServer(1, server2.RemoteHost, server2.Port, 157);
writer.SetServer(2, server3.RemoteHost, server3.Port, 157);
writer.Save();
var loader = new TestDataLoader(pageSize);
var parser = new TestIntParser();
parser.SetCommandsHandler(
new UserCommandsHandler <TestCommand, Type, TestCommand, int, int, TestDbReader>(
new TestUserCommandCreator(), new TestMetaDataCommandCreator()));
var merge = new OrderMerge(loader, parser);
var async = new AsyncTaskModule(new QueueConfiguration(4, 10));
var distributor =
new DistributorModule(new CollectorModel(new DistributorHashConfiguration(1),
new HashMapConfiguration("TestCollector4", HashMapCreationMode.ReadFromFile, 1, 1,
HashFileType.Writer)), async, new AsyncTasksConfiguration(TimeSpan.FromMinutes(1)));
var back = new BackgroundModule(new QueueConfiguration(5, 10));
var searchModule = new SearchTaskModule("", merge, loader, distributor, back, parser);
#region hell
loader.Data.Add(server1, new List <SearchData>
{
TestHelper.CreateData(1),
TestHelper.CreateData(2),
TestHelper.CreateData(4),
TestHelper.CreateData(5),
TestHelper.CreateData(6),
TestHelper.CreateData(7),
TestHelper.CreateData(8),
});
loader.Data.Add(server2, new List <SearchData>
{
TestHelper.CreateData(4),
TestHelper.CreateData(5),
TestHelper.CreateData(6),
TestHelper.CreateData(7),
TestHelper.CreateData(8),
TestHelper.CreateData(9),
TestHelper.CreateData(10),
TestHelper.CreateData(11),
});
loader.Data.Add(server3, new List <SearchData>
{
TestHelper.CreateData(2),
TestHelper.CreateData(3),
TestHelper.CreateData(5),
TestHelper.CreateData(7),
TestHelper.CreateData(8),
TestHelper.CreateData(9),
TestHelper.CreateData(10),
TestHelper.CreateData(11),
TestHelper.CreateData(12),
TestHelper.CreateData(13),
});
#endregion
searchModule.Start();
distributor.Start();
merge.Start();
back.Start();
async.Start();
var reader = searchModule.CreateReader("asc", -1, 5);
reader.Start();
const int count = 13;
for (int i = 0; i < count; i++)
{
Assert.IsTrue(reader.IsCanRead);
reader.ReadNext();
Assert.AreEqual(i + 1, reader.GetValue(0));
}
reader.ReadNext();
Assert.IsFalse(reader.IsCanRead);
reader.Dispose();
async.Dispose();
back.Dispose();
}
