private static ClusterVNodeSettings CreateVNode(int nodeNumber)
{
int tcpIntPort = StartingPort + nodeNumber * 2,
tcpExtPort = tcpIntPort + 1,
httpIntPort = tcpIntPort + 10,
httpExtPort = tcpIntPort + 11;
var vnode = new ClusterVNodeSettings(Guid.NewGuid(), 0,
GetLoopbackForPort(tcpIntPort), null,
GetLoopbackForPort(tcpExtPort), null,
GetLoopbackForPort(httpIntPort), GetLoopbackForPort(httpExtPort),
new Data.GossipAdvertiseInfo(GetLoopbackForPort(tcpIntPort),null,
GetLoopbackForPort(tcpExtPort),null,
GetLoopbackForPort(httpIntPort),
GetLoopbackForPort(httpExtPort)),
new[] { GetLoopbackForPort(httpIntPort).ToHttpUrl() },
new[] { GetLoopbackForPort(httpExtPort).ToHttpUrl() },
false, null, 1, false, "dns", new[] { GetLoopbackForPort(ManagerPort) },
TFConsts.MinFlushDelayMs, 3, 2, 2, TimeSpan.FromSeconds(2),
TimeSpan.FromSeconds(2), false, null, false, TimeSpan.FromHours(1),
StatsStorage.StreamAndCsv, 0, new InternalAuthenticationProviderFactory(), false, 30, true, true, true,
TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1),
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(10), true, Opts.MaxMemtableSizeDefault, false, false, false);
return vnode;
}
public ClusterSettings(string clusterDns, IPEndPoint clusterManagerEndPoint, int clusterNodesCount)
{
Ensure.NotNullOrEmpty(clusterDns, "clusterDns");
Ensure.NotNull(clusterManagerEndPoint, "clusterManagerEndPoint");
ClusterDns = clusterDns;
GroupMembers = new ClusterVNodeSettings[0];
ClusterManager = clusterManagerEndPoint;
ClusterNodesCount = clusterNodesCount;
}
public virtual void TestFixtureSetUp()
{
_builder = TestVNodeBuilder.AsSingleNode()
.RunInMemory();
Given();
_node = _builder.Build();
_settings = ((TestVNodeBuilder)_builder).GetSettings();
_dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig();
_node.Start();
}
public virtual void TestFixtureSetUp()
{
_builder = TestVNodeBuilder.AsClusterMember(_clusterSize)
.RunInMemory();
_quorumSize = _clusterSize / 2 + 1;
Given();
_node = _builder.Build();
_settings = ((TestVNodeBuilder)_builder).GetSettings();
_dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig();
_node.Start();
}
public ClusterSettings(string clusterDns,
IPEndPoint clusterManager,
ClusterVNodeSettings self,
ClusterVNodeSettings[] groupMembers,
int expectedNodesCount)
{
if (string.IsNullOrWhiteSpace(clusterDns))
throw new ArgumentException(string.Format("Wrong cluster DNS name: {0}", clusterDns), clusterDns);
if (self == null)
throw new ArgumentNullException("self");
if (groupMembers == null)
throw new ArgumentNullException("groupMembers");
if (clusterManager == null)
throw new ArgumentNullException("clusterManager");
ClusterDns = clusterDns;
Self = self;
GroupMembers = groupMembers;
ClusterManager = clusterManager;
ClusterNodesCount = expectedNodesCount;
}
/// <summary>
/// Converts an <see cref="EmbeddedVNodeBuilder"/> to a <see cref="ClusterVNode"/>.
/// </summary>
public ClusterVNode Build() {
EnsureHttpPrefixes();
SetUpProjectionsIfNeeded();
var dbConfig = CreateDbConfig(_chunkSize, _dbPath, _chunksCacheSize,
_inMemoryDb);
var db = new TFChunkDb(dbConfig);
var vNodeSettings = new ClusterVNodeSettings(Guid.NewGuid(),
0,
_internalTcp,
_internalSecureTcp,
_externalTcp,
_externalSecureTcp,
_internalHttp,
_externalHttp,
_httpPrefixes.ToArray(),
_enableTrustedAuth,
_certificate,
_workerThreads,
_discoverViaDns,
_clusterDns,
_gossipSeeds.ToArray(),
_minFlushDelay,
_clusterNodeCount,
_prepareAckCount,
_commitAckCount,
_prepareTimeout,
_commitTimeout,
_useSsl,
_sslTargetHost,
_sslValidateServer,
_statsPeriod,
_statsStorage,
_nodePriority,
_authenticationProviderFactory,
_disableScavengeMerging,
_adminOnPublic,
_statsOnPublic,
_gossipOnPublic,
_gossipInterval,
_gossipAllowedTimeDifference,
_gossipTimeout,
_intTcpHeartbeatTimeout,
_intTcpHeartbeatInterval,
_extTcpHeartbeatTimeout,
_extTcpHeartbeatInterval,
!_skipVerifyDbHashes,
_maxMemtableSize);
var infoController = new InfoController(null);
return new ClusterVNode(db, vNodeSettings, GetGossipSource(), infoController, _subsystems.ToArray());
}
public MiniNode(string pathname,
int? tcpPort = null, int? tcpSecPort = null, int? httpPort = null,
ISubsystem[] subsystems = null,
int? chunkSize = null, int? cachedChunkSize = null, bool enableTrustedAuth = false, bool skipInitializeStandardUsersCheck = true,
int memTableSize = 1000,
bool inMemDb = true, bool disableFlushToDisk = false)
{
if (_running) throw new Exception("Previous MiniNode is still running!!!");
_running = true;
RunningTime.Start();
RunCount += 1;
IPAddress ip = IPAddress.Loopback; //GetLocalIp();
int extTcpPort = tcpPort ?? PortsHelper.GetAvailablePort(ip);
int extSecTcpPort = tcpSecPort ?? PortsHelper.GetAvailablePort(ip);
int extHttpPort = httpPort ?? PortsHelper.GetAvailablePort(ip);
int intTcpPort = PortsHelper.GetAvailablePort(ip);
int intSecTcpPort = PortsHelper.GetAvailablePort(ip);
int intHttpPort = PortsHelper.GetAvailablePort(ip);
_dbPath = Path.Combine(pathname, string.Format("mini-node-db-{0}-{1}-{2}", extTcpPort, extSecTcpPort, extHttpPort));
Directory.CreateDirectory(_dbPath);
FileStreamExtensions.ConfigureFlush(disableFlushToDisk);
Db = new TFChunkDb(CreateDbConfig(chunkSize ?? ChunkSize, _dbPath, cachedChunkSize ?? CachedChunkSize, inMemDb));
TcpEndPoint = new IPEndPoint(ip, extTcpPort);
TcpSecEndPoint = new IPEndPoint(ip, extSecTcpPort);
HttpEndPoint = new IPEndPoint(ip, extHttpPort);
IntTcpEndPoint = new IPEndPoint(ip,intTcpPort);
IntSecTcpEndPoint = new IPEndPoint(ip, intSecTcpPort);
IntHttpEndPoint = new IPEndPoint(ip, intHttpPort);
var vNodeSettings = new ClusterVNodeSettings(Guid.NewGuid(),
0,
IntTcpEndPoint,
IntSecTcpEndPoint,
TcpEndPoint,
TcpSecEndPoint,
IntHttpEndPoint,
HttpEndPoint,
new [] {HttpEndPoint.ToHttpUrl()},
enableTrustedAuth,
ssl_connections.GetCertificate(),
1,
false,
"whatever",
new IPEndPoint[] {},
TFConsts.MinFlushDelayMs,
1,
1,
1,
TimeSpan.FromSeconds(2),
TimeSpan.FromSeconds(2),
false,
"",
false,
TimeSpan.FromHours(1),
StatsStorage.None,
1,
new InternalAuthenticationProviderFactory(),
true,
true,
true,
false,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(10),
false,
memTableSize);
Log.Info("\n{0,-25} {1} ({2}/{3}, {4})\n"
+ "{5,-25} {6} ({7})\n"
+ "{8,-25} {9} ({10}-bit)\n"
+ "{11,-25} {12}\n"
+ "{13,-25} {14}\n"
+ "{15,-25} {16}\n"
+ "{17,-25} {18}\n"
+ "{19,-25} {20}\n\n",
"ES VERSION:", VersionInfo.Version, VersionInfo.Branch, VersionInfo.Hashtag, VersionInfo.Timestamp,
"OS:", OS.OsFlavor, Environment.OSVersion,
"RUNTIME:", OS.GetRuntimeVersion(), Marshal.SizeOf(typeof(IntPtr)) * 8,
"GC:", GC.MaxGeneration == 0 ? "NON-GENERATION (PROBABLY BOEHM)" : string.Format("{0} GENERATIONS", GC.MaxGeneration + 1),
"DBPATH:", _dbPath,
"TCP ENDPOINT:", TcpEndPoint,
"TCP SECURE ENDPOINT:", TcpSecEndPoint,
"HTTP ENDPOINT:", HttpEndPoint);
Node = new ClusterVNode(Db, vNodeSettings, new KnownEndpointGossipSeedSource(new [] {HttpEndPoint}), subsystems);
Node.ExternalHttpService.SetupController(new TestController(Node.MainQueue));
}
public ClusterSettings(string clusterDns, IPEndPoint clusterManager, ClusterVNodeSettings self, int clusterNodesCount)
: this(clusterDns, clusterManager, self, new ClusterVNodeSettings[0], clusterNodesCount)
{
}
public MiniClusterNode(
string pathname, int debugIndex, IPEndPoint internalTcp, IPEndPoint internalTcpSec, IPEndPoint internalHttp,
IPEndPoint externalTcp, IPEndPoint externalTcpSec, IPEndPoint externalHttp, IPEndPoint[] gossipSeeds,
ISubsystem[] subsystems = null, int? chunkSize = null, int? cachedChunkSize = null,
bool enableTrustedAuth = false, bool skipInitializeStandardUsersCheck = true, int memTableSize = 1000,
bool inMemDb = true, bool disableFlushToDisk = false)
{
RunningTime.Start();
RunCount += 1;
_dbPath = Path.Combine(
pathname,
string.Format(
"mini-cluster-node-db-{0}-{1}-{2}", externalTcp.Port, externalTcpSec.Port, externalHttp.Port));
Directory.CreateDirectory(_dbPath);
FileStreamExtensions.ConfigureFlush(disableFlushToDisk);
Db =
new TFChunkDb(
CreateDbConfig(chunkSize ?? ChunkSize, _dbPath, cachedChunkSize ?? CachedChunkSize, inMemDb));
InternalTcpEndPoint = internalTcp;
InternalTcpSecEndPoint = internalTcpSec;
InternalHttpEndPoint = internalHttp;
ExternalTcpEndPoint = externalTcp;
ExternalTcpSecEndPoint = externalTcpSec;
ExternalHttpEndPoint = externalHttp;
var singleVNodeSettings = new ClusterVNodeSettings(
Guid.NewGuid(), debugIndex, InternalTcpEndPoint, InternalTcpSecEndPoint, ExternalTcpEndPoint,
ExternalTcpSecEndPoint, InternalHttpEndPoint, ExternalHttpEndPoint,
new[] {ExternalHttpEndPoint.ToHttpUrl()}, enableTrustedAuth, ssl_connections.GetCertificate(), 1, false,
"", gossipSeeds, TFConsts.MinFlushDelayMs, 3, 2, 2, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2),
false, "", false, TimeSpan.FromHours(1), StatsStorage.None, 0,
new InternalAuthenticationProviderFactory(), disableScavengeMerging: true, adminOnPublic: true,
statsOnPublic: true, gossipOnPublic: true, gossipInterval: TimeSpan.FromSeconds(1),
gossipAllowedTimeDifference: TimeSpan.FromSeconds(1), gossipTimeout: TimeSpan.FromSeconds(1),
extTcpHeartbeatTimeout: TimeSpan.FromSeconds(10), extTcpHeartbeatInterval: TimeSpan.FromSeconds(10),
intTcpHeartbeatTimeout: TimeSpan.FromSeconds(10), intTcpHeartbeatInterval: TimeSpan.FromSeconds(10),
verifyDbHash: false, maxMemtableEntryCount: memTableSize);
Log.Info(
"\n{0,-25} {1} ({2}/{3}, {4})\n" + "{5,-25} {6} ({7})\n" + "{8,-25} {9} ({10}-bit)\n"
+ "{11,-25} {12}\n" + "{13,-25} {14}\n" + "{15,-25} {16}\n" + "{17,-25} {18}\n" + "{19,-25} {20}\n\n",
"ES VERSION:", VersionInfo.Version, VersionInfo.Branch, VersionInfo.Hashtag, VersionInfo.Timestamp,
"OS:", OS.OsFlavor, Environment.OSVersion, "RUNTIME:", OS.GetRuntimeVersion(),
Marshal.SizeOf(typeof (IntPtr))*8, "GC:",
GC.MaxGeneration == 0
? "NON-GENERATION (PROBABLY BOEHM)"
: string.Format("{0} GENERATIONS", GC.MaxGeneration + 1), "DBPATH:", _dbPath, "ExTCP ENDPOINT:",
ExternalTcpEndPoint, "ExTCP SECURE ENDPOINT:", ExternalTcpSecEndPoint, "ExHTTP ENDPOINT:",
ExternalHttpEndPoint);
Node = new ClusterVNode(Db, singleVNodeSettings, infoController: new InfoController(null), subsystems: subsystems, gossipSeedSource: new KnownEndpointGossipSeedSource(gossipSeeds));
Node.ExternalHttpService.SetupController(new TestController(Node.MainQueue));
}
public ClusterVNode(TFChunkDb db,
ClusterVNodeSettings vNodeSettings,
IGossipSeedSource gossipSeedSource,
bool dbVerifyHashes,
int memTableEntryCount,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
Ensure.NotNull(gossipSeedSource, "gossipSeedSource");
_nodeInfo = vNodeSettings.NodeInfo;
_mainBus = new InMemoryBus("MainBus");
var forwardingProxy = new MessageForwardingProxy();
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_controller = new ClusterVNodeController(_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_nodeInfo.ExternalHttp,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
// STORAGE SUBSYSTEM
db.Open(dbVerifyHashes);
var indexPath = Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
() => new TFReaderLease(readerPool),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb);
var hash = new XXHashUnsafe();
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
hash,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager,
() => readIndex.LastCommitPosition); // subscribes internally
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, tableIndex, hash, readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: !vNodeSettings.DisableScavengeMerging);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// AUTHENTICATION INFRASTRUCTURE - delegate to plugins
var authenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);
Ensure.NotNull(authenticationProvider, "authenticationProvider");
{
// EXTERNAL TCP
var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
authenticationProvider, null);
_mainBus.Subscribe<SystemMessage.SystemInit>(extTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(extTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(extTcpService);
// EXTERNAL SECURE TCP
if (_nodeInfo.ExternalSecureTcp != null)
{
var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
authenticationProvider, vNodeSettings.Certificate);
_mainBus.Subscribe<SystemMessage.SystemInit>(extSecTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(extSecTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(extSecTcpService);
}
// INTERNAL TCP
var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Normal,
new InternalTcpDispatcher(),
ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
authenticationProvider, null);
_mainBus.Subscribe<SystemMessage.SystemInit>(intTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(intTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(intTcpService);
// INTERNAL SECURE TCP
if (_nodeInfo.InternalSecureTcp != null)
{
var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Secure,
new InternalTcpDispatcher(),
ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
authenticationProvider, vNodeSettings.Certificate);
_mainBus.Subscribe<SystemMessage.SystemInit>(intSecTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(intSecTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(intSecTcpService);
}
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(authenticationProvider),
};
if (vNodeSettings.EnableTrustedAuth)
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
});
var adminController = new AdminController(_mainQueue);
var pingController = new PingController();
var statController = new StatController(monitoringQueue, _workersHandler);
var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler);
var gossipController = new GossipController(_mainQueue, _workersHandler);
var electController = new ElectController(_mainQueue);
// HTTP SENDERS
gossipController.SubscribeSenders(httpPipe);
electController.SubscribeSenders(httpPipe);
// EXTERNAL HTTP
_externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_externalHttpService.SetupController(adminController);
_externalHttpService.SetupController(pingController);
_externalHttpService.SetupController(statController);
_externalHttpService.SetupController(atomController);
_externalHttpService.SetupController(gossipController);
_mainBus.Subscribe<SystemMessage.SystemInit>(_externalHttpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_externalHttpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
// INTERNAL HTTP
_internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
_workersHandler, _nodeInfo.InternalHttp.ToHttpUrl());
_internalHttpService.SetupController(adminController);
_internalHttpService.SetupController(pingController);
_internalHttpService.SetupController(statController);
_internalHttpService.SetupController(atomController);
_internalHttpService.SetupController(gossipController);
_internalHttpService.SetupController(electController);
// Authentication plugin HTTP
vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);
_mainBus.Subscribe<SystemMessage.SystemInit>(_internalHttpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_internalHttpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST FORWARDING
var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
_mainBus.Subscribe<SystemMessage.SystemStart>(forwardingService);
_mainBus.Subscribe<SystemMessage.RequestForwardingTimerTick>(forwardingService);
_mainBus.Subscribe<ClientMessage.NotHandled>(forwardingService);
_mainBus.Subscribe<ClientMessage.WriteEventsCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.TransactionStartCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.TransactionWriteCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.TransactionCommitCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.DeleteStreamCompleted>(forwardingService);
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue,
vNodeSettings.PrepareAckCount,
vNodeSettings.CommitAckCount,
vNodeSettings.PrepareTimeout,
vNodeSettings.CommitTimeout);
_mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe<TimerMessage.Schedule>(_timerService);
// MASTER REPLICATION
var masterReplicationService = new MasterReplicationService(_mainQueue, _nodeInfo.InstanceId, db, _workersHandler,
epochManager, vNodeSettings.ClusterNodeCount);
_mainBus.Subscribe<SystemMessage.SystemStart>(masterReplicationService);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(masterReplicationService);
_mainBus.Subscribe<ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
_mainBus.Subscribe<ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);
// REPLICA REPLICATION
var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, authenticationProvider,
_nodeInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(replicaService);
_mainBus.Subscribe<ReplicationMessage.ReconnectToMaster>(replicaService);
_mainBus.Subscribe<ReplicationMessage.SubscribeToMaster>(replicaService);
_mainBus.Subscribe<ReplicationMessage.AckLogPosition>(replicaService);
_mainBus.Subscribe<StorageMessage.PrepareAck>(replicaService);
_mainBus.Subscribe<StorageMessage.CommitAck>(replicaService);
_mainBus.Subscribe<ClientMessage.TcpForwardMessage>(replicaService);
// ELECTIONS
var electionsService = new ElectionsService(_mainQueue, _nodeInfo, vNodeSettings.ClusterNodeCount,
db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
electionsService.SubscribeMessages(_mainBus);
// GOSSIP
var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, _nodeInfo, db.Config.WriterCheckpoint,
db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
vNodeSettings.NodePriority);
_mainBus.Subscribe<SystemMessage.SystemInit>(gossip);
_mainBus.Subscribe<GossipMessage.RetrieveGossipSeedSources>(gossip);
_mainBus.Subscribe<GossipMessage.GotGossipSeedSources>(gossip);
_mainBus.Subscribe<GossipMessage.Gossip>(gossip);
_mainBus.Subscribe<GossipMessage.GossipReceived>(gossip);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(gossip);
_mainBus.Subscribe<GossipMessage.GossipSendFailed>(gossip);
_mainBus.Subscribe<SystemMessage.VNodeConnectionEstablished>(gossip);
_mainBus.Subscribe<SystemMessage.VNodeConnectionLost>(gossip);
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
{
foreach (var subsystem in subsystems)
{
subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider,
httpSendService, new[] { _internalHttpService, _externalHttpService }, _workersHandler);
}
}
}
/// <summary>
/// Converts an <see cref="VNodeBuilder"/> to a <see cref="ClusterVNode"/>.
/// </summary>
/// <param name="options">The options with which to build the infoController</param>
/// <param name="consumerStrategies">The consumer strategies with which to build the node</param>
/// <returns>A <see cref="ClusterVNode"/> built with the options that were set on the <see cref="VNodeBuilder"/></returns>
public ClusterVNode Build(IOptions options = null, IPersistentSubscriptionConsumerStrategyFactory[] consumerStrategies = null)
{
EnsureHttpPrefixes();
SetUpProjectionsIfNeeded();
_gossipAdvertiseInfo = EnsureGossipAdvertiseInfo();
_dbConfig = CreateDbConfig(_chunkSize, _cachedChunks, _dbPath, _chunksCacheSize,
_inMemoryDb, _log);
FileStreamExtensions.ConfigureFlush(disableFlushToDisk: _unsafeDisableFlushToDisk);
_db = new TFChunkDb(_dbConfig);
_vNodeSettings = new ClusterVNodeSettings(Guid.NewGuid(),
0,
_internalTcp,
_internalSecureTcp,
_externalTcp,
_externalSecureTcp,
_internalHttp,
_externalHttp,
_gossipAdvertiseInfo,
_intHttpPrefixes.ToArray(),
_extHttpPrefixes.ToArray(),
_enableTrustedAuth,
_certificate,
_workerThreads,
_discoverViaDns,
_clusterDns,
_gossipSeeds.ToArray(),
_minFlushDelay,
_clusterNodeCount,
_prepareAckCount,
_commitAckCount,
_prepareTimeout,
_commitTimeout,
_useSsl,
_sslTargetHost,
_sslValidateServer,
_statsPeriod,
_statsStorage,
_nodePriority,
_authenticationProviderFactory,
_disableScavengeMerging,
_scavengeHistoryMaxAge,
_adminOnPublic,
_statsOnPublic,
_gossipOnPublic,
_gossipInterval,
_gossipAllowedTimeDifference,
_gossipTimeout,
_intTcpHeartbeatTimeout,
_intTcpHeartbeatInterval,
_extTcpHeartbeatTimeout,
_extTcpHeartbeatInterval,
!_skipVerifyDbHashes,
_maxMemtableSize,
_hashCollisionReadLimit,
_startStandardProjections,
_disableHTTPCaching,
_logHttpRequests,
_index,
_enableHistograms,
_indexCacheDepth,
_indexBitnessVersion,
consumerStrategies,
_unsafeIgnoreHardDelete,
_betterOrdering,
_readerThreadsCount,
_alwaysKeepScavenged);
var infoController = new InfoController(options, _projectionType);
_log.Info("{0,-25} {1}", "INSTANCE ID:", _vNodeSettings.NodeInfo.InstanceId);
_log.Info("{0,-25} {1}", "DATABASE:", _db.Config.Path);
_log.Info("{0,-25} {1} (0x{1:X})", "WRITER CHECKPOINT:", _db.Config.WriterCheckpoint.Read());
_log.Info("{0,-25} {1} (0x{1:X})", "CHASER CHECKPOINT:", _db.Config.ChaserCheckpoint.Read());
_log.Info("{0,-25} {1} (0x{1:X})", "EPOCH CHECKPOINT:", _db.Config.EpochCheckpoint.Read());
_log.Info("{0,-25} {1} (0x{1:X})", "TRUNCATE CHECKPOINT:", _db.Config.TruncateCheckpoint.Read());
return new ClusterVNode(_db, _vNodeSettings, GetGossipSource(), infoController, _subsystems.ToArray());
}
