private void ReOpenDb()
{
Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, WriterCheckpoint, ChaserCheckpoint));
Db.Open();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
() => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV3, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.IndexV3,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: Db.Config.ReplicationCheckpoint);
((ReadIndex)ReadIndex).IndexCommitter.Init(ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close(removeFiles: false);
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV2, maxSize: MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.IndexV2,
5,
maxSizeForMemory: MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: 1,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: Db.Config.ReplicationCheckpoint);
ReadIndex.Init(ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.IndexV3,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault, Opts.SkipIndexScanOnReadsDefault, _dbResult.Db.Config.ReplicationCheckpoint);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
var scavenger = new TFChunkScavenger(_dbResult.Db, new FakeTFScavengerLog(), tableIndex, ReadIndex, unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false).Wait();
}
private void ReOpenDb()
{
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(ChaserCheckpoint.Read());
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
ReplicationCheckpoint = new InMemoryCheckpoint(-1);
Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, WriterCheckpoint, ChaserCheckpoint,
replicationCheckpoint: ReplicationCheckpoint));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
() => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: Db.Config.ReplicationCheckpoint);
ReadIndex.Init(ChaserCheckpoint.Read());
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
{
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
}
_scavenger = new TFChunkScavenger(Db, new FakeTFScavengerLog(), TableIndex, ReadIndex);
await _scavenger.Scavenge(alwaysKeepScavenged : true, mergeChunks : _mergeChunks);
}
}
public void ConstructTableIndexWithCorruptIndexEntries(byte version, bool skipIndexVerify, bool createForceVerifyFile = false)
{
base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var fakeReader = new TFReaderLease(new FakeIndexReader());
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
5,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
//create index entries
for (int i = 1; i <= NumIndexEntries; i++)
{
_tableIndex.Add(i * 1337, StreamName, i, i * 1337);
}
_tableIndex.Close(false);
//load index map to obtain ptable filenames
_indexMap = IndexMap.FromFile(Path.Combine(PathName, TableIndex.IndexMapFilename));
List <string> ptableFiles = new List <string>();
foreach (string ptableFilename in _indexMap.GetAllFilenames())
{
ptableFiles.Add(ptableFilename);
}
_indexMap.Dispose(TimeSpan.FromSeconds(5));
//corrupt ptable files
foreach (string ptableFilename in ptableFiles)
{
CorruptPTableFile(ptableFilename, version, "zeroOutMiddleEntry");
}
//create force verify file if requested
if (createForceVerifyFile)
{
using (FileStream fs = new FileStream(Path.Combine(PathName, TableIndex.ForceIndexVerifyFilename), FileMode.OpenOrCreate)){
};
}
//load table index again
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
5,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify,
indexCacheDepth: 8);
_tableIndex.Initialize(long.MaxValue);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 20),
() => { throw new InvalidOperationException(); },
_ptableVersion,
maxSizeForMemory: 10);
_tableIndex.Initialize(long.MaxValue);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 20),
() => { throw new InvalidOperationException(); },
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 10,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
}
public void CreateDb(params Rec[] records)
{
if (DbRes != null)
{
DbRes.Db.Close();
}
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024 * 1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1),
inMemDb: true);
var dbHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = dbHelper.Chunk(records).CreateDb();
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>(
"Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV3, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.IndexV3,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault);
ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024 * 1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.IndexV3,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var bus = new InMemoryBus("Bus");
var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp",
unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public void CreateDb(params Rec[] records)
{
if (DbRes != null)
{
DbRes.Db.Close();
}
var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = dbHelper.Chunk(records).CreateDb();
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>(
"Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV3, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.IndexV3,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: DbRes.Db.Config.ReplicationCheckpoint,
indexCheckpoint: DbRes.Db.Config.IndexCheckpoint);
((ReadIndex)ReadIndex).IndexCommitter.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024 * 1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>(
"Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024 * 1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: 200),
() => new TFReaderLease(readerPool),
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
var hasher = new XXHashUnsafe();
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, hasher, 100, true, _metastreamMaxCount);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var scavenger = new TFChunkScavenger(_dbResult.Db, tableIndex, hasher, ReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>(
"Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV2, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.IndexV2,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: DbRes.Db.Config.ReplicationCheckpoint);
ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
}
private static void TestHashes()
{
var s = new Stopwatch();
s.Start();
for (int i = 0; i < 10000000; i++)
{
var y = i.ToString("00000000000000000000000").GetHashCode();
}
Console.WriteLine(".net " + s.Elapsed);
s.Reset();
s.Start();
var hasher = new Murmur2Unsafe();
for (int i = 0; i < 10000000; i++)
{
hasher.Hash(i.ToString("00000000000000000000000"));
}
Console.WriteLine("mm2a " + s.Elapsed);
s.Reset();
s.Start();
var hasher2 = new Murmur3AUnsafe();
for (int i = 0; i < 10000000; i++)
{
hasher2.Hash(i.ToString("00000000000000000000000"));
}
Console.WriteLine("mm3a " + s.Elapsed);
s.Reset();
s.Start();
var hasher3 = new XXHashUnsafe();
for (int i = 0; i < 10000000; i++)
{
hasher3.Hash(i.ToString("00000000000000000000000"));
}
Console.WriteLine("xx " + s.Elapsed);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var writerCheckpoint = new InMemoryCheckpoint(0);
var chaserCheckpoint = new InMemoryCheckpoint(0);
var bus = new InMemoryBus("bus");
new IODispatcher(bus, new PublishEnvelope(bus));
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerCheckpoint,
chaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
// create db
Writer = new TFChunkWriter(_db);
Writer.Open();
SetupDB();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(_db, _db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
_tableIndex,
EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault);
ReadIndex.Init(chaserCheckpoint.Read());
_tableIndex.Close(false);
Writer = new TFChunkWriter(_db);
Writer.Open();
Given();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
_tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
_tableIndex,
EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault);
ReadIndex.Init(chaserCheckpoint.Read());
}
public static byte[] WithHash(this byte[] self)
{
var hasher = new XXHashUnsafe();
var hash = hasher.Hash(self.AsSpan()[..^ 4]);
public ClusterVNode(TFChunkDb db,
ClusterVNodeSettings vNodeSettings,
IGossipSeedSource gossipSeedSource,
InfoController infoController,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
Ensure.NotNull(gossipSeedSource, "gossipSeedSource");
var isSingleNode = vNodeSettings.ClusterNodeCount == 1;
_nodeInfo = vNodeSettings.NodeInfo;
_mainBus = new InMemoryBus("MainBus");
var forwardingProxy = new MessageForwardingProxy();
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_controller = new ClusterVNodeController(_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
//SELF
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(this);
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_nodeInfo.ExternalHttp,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
// STORAGE SUBSYSTEM
db.Open(vNodeSettings.VerifyDbHash);
var indexPath = Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: vNodeSettings.MaxMemtableEntryCount * 2),
() => new TFReaderLease(readerPool),
maxSizeForMemory: vNodeSettings.MaxMemtableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb);
var hash = new XXHashUnsafe();
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
hash,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager,
() => readIndex.LastCommitPosition); // subscribes internally
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
#if DEBUG
QueueStatsCollector.InitializeCheckpoints(
_nodeInfo.DebugIndex, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
#endif
_mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, tableIndex, hash, readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: !vNodeSettings.DisableScavengeMerging);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// AUTHENTICATION INFRASTRUCTURE - delegate to plugins
var authenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);
Ensure.NotNull(authenticationProvider, "authenticationProvider");
{
// EXTERNAL TCP
var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
authenticationProvider, null);
_mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService);
// EXTERNAL SECURE TCP
if (_nodeInfo.ExternalSecureTcp != null)
{
var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
authenticationProvider, vNodeSettings.Certificate);
_mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService);
}
if (!isSingleNode)
{
// INTERNAL TCP
var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Normal,
new InternalTcpDispatcher(),
vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
authenticationProvider, null);
_mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService);
// INTERNAL SECURE TCP
if (_nodeInfo.InternalSecureTcp != null)
{
var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Secure,
new InternalTcpDispatcher(),
vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
authenticationProvider, vNodeSettings.Certificate);
_mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService);
}
}
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
var httpAuthenticationProviders = new List <HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(authenticationProvider),
};
if (vNodeSettings.EnableTrustedAuth)
{
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
}
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
});
var adminController = new AdminController(_mainQueue);
var pingController = new PingController();
var statController = new StatController(monitoringQueue, _workersHandler);
var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler);
var gossipController = new GossipController(_mainQueue, _workersHandler, vNodeSettings.GossipTimeout);
var electController = new ElectController(_mainQueue);
// HTTP SENDERS
gossipController.SubscribeSenders(httpPipe);
electController.SubscribeSenders(httpPipe);
// EXTERNAL HTTP
_externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
if (vNodeSettings.AdminOnPublic)
{
_externalHttpService.SetupController(adminController);
}
_externalHttpService.SetupController(pingController);
_externalHttpService.SetupController(infoController);
if (vNodeSettings.StatsOnPublic)
{
_externalHttpService.SetupController(statController);
}
_externalHttpService.SetupController(atomController);
if (vNodeSettings.GossipOnPublic)
{
_externalHttpService.SetupController(gossipController);
}
_mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
// INTERNAL HTTP
if (!isSingleNode)
{
_internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
_workersHandler, _nodeInfo.InternalHttp.ToHttpUrl());
_internalHttpService.SetupController(adminController);
_internalHttpService.SetupController(pingController);
_internalHttpService.SetupController(infoController);
_internalHttpService.SetupController(statController);
_internalHttpService.SetupController(atomController);
_internalHttpService.SetupController(gossipController);
_internalHttpService.SetupController(electController);
}
// Authentication plugin HTTP
vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);
if (_internalHttpService != null)
{
_mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
}
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST FORWARDING
var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
_mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService);
_mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService);
_mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService);
_mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService);
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue,
vNodeSettings.PrepareAckCount,
vNodeSettings.CommitAckCount,
vNodeSettings.PrepareTimeout,
vNodeSettings.CommitTimeout);
_mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe <SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe <TimerMessage.Schedule>(_timerService);
if (!isSingleNode)
{
// MASTER REPLICATION
var masterReplicationService = new MasterReplicationService(_mainQueue, _nodeInfo.InstanceId, db, _workersHandler,
epochManager, vNodeSettings.ClusterNodeCount);
_mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService);
_mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
_mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);
// REPLICA REPLICATION
var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, authenticationProvider,
_nodeInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer,
vNodeSettings.IntTcpHeartbeatTimeout, vNodeSettings.ExtTcpHeartbeatInterval);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService);
_mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService);
_mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService);
_mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService);
_mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService);
_mainBus.Subscribe <StorageMessage.CommitAck>(replicaService);
_mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService);
}
// ELECTIONS
var electionsService = new ElectionsService(_mainQueue, _nodeInfo, vNodeSettings.ClusterNodeCount,
db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
electionsService.SubscribeMessages(_mainBus);
if (!isSingleNode)
{
// GOSSIP
var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, _nodeInfo, db.Config.WriterCheckpoint,
db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
vNodeSettings.NodePriority, vNodeSettings.GossipInterval, vNodeSettings.GossipAllowedTimeDifference);
_mainBus.Subscribe <SystemMessage.SystemInit>(gossip);
_mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip);
_mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip);
_mainBus.Subscribe <GossipMessage.Gossip>(gossip);
_mainBus.Subscribe <GossipMessage.GossipReceived>(gossip);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip);
_mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip);
_mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip);
_mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip);
}
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
{
foreach (var subsystem in subsystems)
{
var http = isSingleNode ? new [] { _externalHttpService } : new [] { _internalHttpService, _externalHttpService };
subsystem.Register(new StandardComponents(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, http, _workersHandler));
}
}
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
Bus = new InMemoryBus("bus");
IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(ChaserCheckpoint.Read());
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
{
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
}
_scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp");
_scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
}
}
public SingleVNode(TFChunkDb db,
SingleVNodeSettings vNodeSettings,
bool dbVerifyHashes,
int memTableEntryCount,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
_settings = vNodeSettings;
_mainBus = new InMemoryBus("MainBus");
_controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_settings.ExternalHttpEndPoint,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
db.Open(dbVerifyHashes);
// STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
() => new TFReaderLease(readerPool),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb);
var hasher = new XXHashUnsafe();
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
hasher,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager); // subscribes internally
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
#if DEBUG
QueueStatsCollector.InitializeCheckpoints(-1, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
#endif
_mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db,
tableIndex,
hasher,
readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: !vNodeSettings.DisableScavengeMerging);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
// AUTHENTICATION INFRASTRUCTURE
var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
var dispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_workersHandler, crossThread: true));
var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
_mainBus.Subscribe <SystemMessage.SystemStart>(passwordChangeNotificationReader);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
_mainBus.Subscribe(internalAuthenticationProvider);
SubscribeWorkers(bus =>
{
bus.Subscribe(dispatcher.ForwardReader);
bus.Subscribe(dispatcher.BackwardReader);
bus.Subscribe(dispatcher.Writer);
bus.Subscribe(dispatcher.StreamDeleter);
bus.Subscribe(dispatcher);
});
// TCP
{
var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, null);
_mainBus.Subscribe <SystemMessage.SystemInit>(tcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(tcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(tcpService);
}
// SECURE TCP
if (vNodeSettings.ExternalSecureTcpEndPoint != null)
{
var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, _settings.Certificate);
_mainBus.Subscribe <SystemMessage.SystemInit>(secureTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(secureTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(secureTcpService);
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
// HTTP
var httpAuthenticationProviders = new List <HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
};
if (_settings.EnableTrustedAuth)
{
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
}
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
});
_httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_httpService.SetupController(new AdminController(_mainQueue));
_httpService.SetupController(new PingController());
_httpService.SetupController(new StatController(monitoringQueue, _workersHandler));
_httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
_httpService.SetupController(new GuidController(_mainQueue));
_httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));
_mainBus.Subscribe <SystemMessage.SystemInit>(_httpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_httpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_httpService);
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout);
_mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe <SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription);
// USER MANAGEMENT
var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
_mainBus.Subscribe(ioDispatcher.BackwardReader);
_mainBus.Subscribe(ioDispatcher.ForwardReader);
_mainBus.Subscribe(ioDispatcher.Writer);
_mainBus.Subscribe(ioDispatcher.StreamDeleter);
_mainBus.Subscribe(ioDispatcher);
var userManagement = new UserManagementService(
_mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck);
_mainBus.Subscribe <UserManagementMessage.Create>(userManagement);
_mainBus.Subscribe <UserManagementMessage.Update>(userManagement);
_mainBus.Subscribe <UserManagementMessage.Enable>(userManagement);
_mainBus.Subscribe <UserManagementMessage.Disable>(userManagement);
_mainBus.Subscribe <UserManagementMessage.Delete>(userManagement);
_mainBus.Subscribe <UserManagementMessage.ResetPassword>(userManagement);
_mainBus.Subscribe <UserManagementMessage.ChangePassword>(userManagement);
_mainBus.Subscribe <UserManagementMessage.Get>(userManagement);
_mainBus.Subscribe <UserManagementMessage.GetAll>(userManagement);
_mainBus.Subscribe <SystemMessage.BecomeMaster>(userManagement);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe <TimerMessage.Schedule>(_timerService);
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
{
foreach (var subsystem in subsystems)
{
subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[] { _httpService }, _workersHandler);
}
}
}
