public MiniNode(string pathname)
{
var ip = GetLocalIp();
int extTcpPort = GetAvailablePort(ip);
int extHttpPort = GetAvailablePort(ip);
_dbPath = Path.Combine(pathname, string.Format("mini-node-db-{0}-{1}", extTcpPort, extHttpPort));
Directory.CreateDirectory(_dbPath);
_tfChunkDb = new TFChunkDb(CreateOneTimeDbConfig(1024*1024, _dbPath, 1));
TcpEndPoint = new IPEndPoint(ip, extTcpPort);
HttpEndPoint = new IPEndPoint(ip, extHttpPort);
var singleVNodeSettings = new SingleVNodeSettings(TcpEndPoint,
HttpEndPoint,
new[] {HttpEndPoint.ToHttpUrl()},
1,
1,
1,
TimeSpan.FromHours(1),
StatsStorage.None);
_node = new SingleVNode(_tfChunkDb, singleVNodeSettings, dbVerifyHashes: true, memTableEntryCount: 1000);
}
public MiniClusterNode(
string pathname, int debugIndex, IPEndPoint internalTcp, IPEndPoint internalTcpSec, IPEndPoint internalHttp,
IPEndPoint externalTcp, IPEndPoint externalTcpSec, IPEndPoint externalHttp, IPEndPoint[] gossipSeeds,
ISubsystem[] subsystems = null, int? chunkSize = null, int? cachedChunkSize = null,
bool enableTrustedAuth = false, bool skipInitializeStandardUsersCheck = true, int memTableSize = 1000,
bool inMemDb = true, bool disableFlushToDisk = false)
{
RunningTime.Start();
RunCount += 1;
_dbPath = Path.Combine(
pathname,
string.Format(
"mini-cluster-node-db-{0}-{1}-{2}", externalTcp.Port, externalTcpSec.Port, externalHttp.Port));
Directory.CreateDirectory(_dbPath);
FileStreamExtensions.ConfigureFlush(disableFlushToDisk);
Db =
new TFChunkDb(
CreateDbConfig(chunkSize ?? ChunkSize, _dbPath, cachedChunkSize ?? CachedChunkSize, inMemDb));
InternalTcpEndPoint = internalTcp;
InternalTcpSecEndPoint = internalTcpSec;
InternalHttpEndPoint = internalHttp;
ExternalTcpEndPoint = externalTcp;
ExternalTcpSecEndPoint = externalTcpSec;
ExternalHttpEndPoint = externalHttp;
var singleVNodeSettings = new ClusterVNodeSettings(
Guid.NewGuid(), debugIndex, InternalTcpEndPoint, InternalTcpSecEndPoint, ExternalTcpEndPoint,
ExternalTcpSecEndPoint, InternalHttpEndPoint, ExternalHttpEndPoint,
new Data.GossipAdvertiseInfo(InternalTcpEndPoint, InternalTcpSecEndPoint,
ExternalTcpEndPoint, ExternalTcpSecEndPoint,
InternalHttpEndPoint, ExternalHttpEndPoint),
new[] {InternalHttpEndPoint.ToHttpUrl()}, new[]{ExternalHttpEndPoint.ToHttpUrl()}, enableTrustedAuth, ssl_connections.GetCertificate(), 1, false,
"", gossipSeeds, TFConsts.MinFlushDelayMs, 3, 2, 2, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2),
false, "", false, TimeSpan.FromHours(1), StatsStorage.None, 0,
new InternalAuthenticationProviderFactory(), disableScavengeMerging: true, adminOnPublic: true,
statsOnPublic: true, gossipOnPublic: true, gossipInterval: TimeSpan.FromSeconds(1),
gossipAllowedTimeDifference: TimeSpan.FromSeconds(1), gossipTimeout: TimeSpan.FromSeconds(1),
extTcpHeartbeatTimeout: TimeSpan.FromSeconds(10), extTcpHeartbeatInterval: TimeSpan.FromSeconds(10),
intTcpHeartbeatTimeout: TimeSpan.FromSeconds(10), intTcpHeartbeatInterval: TimeSpan.FromSeconds(10),
verifyDbHash: false, maxMemtableEntryCount: memTableSize, developmentMode: false);
Log.Info(
"\n{0,-25} {1} ({2}/{3}, {4})\n" + "{5,-25} {6} ({7})\n" + "{8,-25} {9} ({10}-bit)\n"
+ "{11,-25} {12}\n" + "{13,-25} {14}\n" + "{15,-25} {16}\n" + "{17,-25} {18}\n" + "{19,-25} {20}\n\n",
"ES VERSION:", VersionInfo.Version, VersionInfo.Branch, VersionInfo.Hashtag, VersionInfo.Timestamp,
"OS:", OS.OsFlavor, Environment.OSVersion, "RUNTIME:", OS.GetRuntimeVersion(),
Marshal.SizeOf(typeof (IntPtr))*8, "GC:",
GC.MaxGeneration == 0
? "NON-GENERATION (PROBABLY BOEHM)"
: string.Format("{0} GENERATIONS", GC.MaxGeneration + 1), "DBPATH:", _dbPath, "ExTCP ENDPOINT:",
ExternalTcpEndPoint, "ExTCP SECURE ENDPOINT:", ExternalTcpSecEndPoint, "ExHTTP ENDPOINT:",
ExternalHttpEndPoint);
Node = new ClusterVNode(Db, singleVNodeSettings, infoController: new InfoController(null, ProjectionType.None), subsystems: subsystems, gossipSeedSource: new KnownEndpointGossipSeedSource(gossipSeeds));
Node.ExternalHttpService.SetupController(new TestController(Node.MainQueue));
}
private void ReOpenDb()
{
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
TableIndex = new TableIndex(Path.Combine(PathName, "index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
new ByLengthHasher(),
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount);
ReadIndex.Init(ChaserCheckpoint.Read());
}
private void ReOpenDb()
{
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
TableIndex = new TableIndex(Path.Combine(PathName, "index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
5,
() => new TFChunkReader(Db, Db.Config.WriterCheckpoint),
TableIndex,
new ByLengthHasher(),
new NoLRUCache<string, StreamCacheInfo>(),
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount);
ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
}
private void ReOpenDb()
{
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
4096,
0,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
var chunk = _db.Manager.GetChunk(0);
_records = new LogRecord[RecordsCount];
_results = new RecordWriteResult[RecordsCount];
for (int i = 0; i < _records.Length; ++i)
{
_records[i] = LogRecord.SingleWrite(i == 0 ? 0 : _results[i - 1].NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_results[i] = chunk.TryAppend(_records[i]);
}
chunk.Flush();
_db.Config.WriterCheckpoint.Write(_results[RecordsCount - 1].NewPosition);
_db.Config.WriterCheckpoint.Flush();
}
public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later()
{
var writerchk = new InMemoryCheckpoint(0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerchk,
new InMemoryCheckpoint()));
db.OpenVerifyAndClean();
var writer = new TFChunkWriter(db);
writer.Open();
var reader = new TFChunkSequentialReader(db, writerchk, 0);
LogRecord record;
Assert.IsFalse(reader.TryReadNext(out record));
var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null);
long tmp;
Assert.IsTrue(writer.Write(rec, out tmp));
writer.Flush();
writer.Close();
Assert.IsTrue(reader.TryReadNext(out record));
Assert.AreEqual(rec, record);
reader.Close();
db.Close();
}
public void SetUp()
{
_writerCheckpoint = new InMemoryCheckpoint();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024,
0,
_writerCheckpoint,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
_writer = new TFChunkWriter(_db);
_writer.Open();
_record = new PrepareLogRecord(logPosition: 0,
eventId: _eventId,
correlationId: _correlationId,
transactionPosition: 0xDEAD,
transactionOffset: 0xBEEF,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.SingleWrite,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] {7, 17});
long newPos;
_writer.Write(_record, out newPos);
_writer.Flush();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
16 * 1024,
0,
new InMemoryCheckpoint(),
new ICheckpoint[0]));
_db.OpenVerifyAndClean();
var chunk = _db.Manager.GetChunk(0);
_rec1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res1 = chunk.TryAppend(_rec1);
_rec2 = LogRecord.SingleWrite(_res1.NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res2 = chunk.TryAppend(_rec2);
_rec3 = LogRecord.SingleWrite(_res2.NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res3 = chunk.TryAppend(_rec3);
chunk.Complete();
var scavenger = new TFChunkScavenger(_db, new FakeReadIndex(x => x == "es-to-scavenge"));
scavenger.Scavenge(alwaysKeepScavenged: true);
_scavengedChunk = _db.Manager.GetChunk(0);
}
public TFChunkScavenger(TFChunkDb db, IReadIndex readIndex)
{
_db = db;
_readIndex = readIndex;
Ensure.NotNull(db, "db");
Ensure.NotNull(readIndex, "readIndex");
}
public Projections(
TFChunkDb db, QueuedHandler mainQueue, InMemoryBus mainBus, TimerService timerService,
HttpService httpService, int projectionWorkerThreadCount)
{
_projectionWorkerThreadCount = projectionWorkerThreadCount;
SetupMessaging(db, mainQueue, mainBus, timerService, httpService);
}
public void try_read_returns_false_when_writer_checksum_is_equal_to_reader_checksum()
{
var writerchk = new InMemoryCheckpoint();
var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
10000,
0,
writerchk,
chaserchk,
new[] {writerchk, chaserchk}));
db.OpenVerifyAndClean();
writerchk.Write(12);
writerchk.Flush();
chaserchk.Write(12);
chaserchk.Flush();
var chaser = new TFChunkChaser(db, writerchk, chaserchk);
chaser.Open();
LogRecord record;
Assert.IsFalse(chaser.TryReadNext(out record));
Assert.AreEqual(12, chaserchk.Read());
chaser.Close();
db.Dispose();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
Bus = new InMemoryBus("bus");
IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(ChaserCheckpoint.Read());
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
_scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp");
_scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
}
}
public ProjectionWorkerNode(TFChunkDb db, QueuedHandler inputQueue)
{
Ensure.NotNull(db, "db");
_coreOutput = new InMemoryBus("Core Output");
_projectionCoreService = new ProjectionCoreService(CoreOutput, inputQueue, 10, db.Config.WriterCheckpoint);
}
public void Register(
TFChunkDb db, QueuedHandler mainQueue, ISubscriber mainBus, TimerService timerService,
ITimeProvider timeProvider, IHttpForwarder httpForwarder, HttpService[] httpServices, IPublisher networkSendService)
{
_projections = new EventStore.Projections.Core.Projections(
db, mainQueue, mainBus, timerService, timeProvider, httpForwarder, httpServices, networkSendService,
projectionWorkerThreadCount: _projectionWorkerThreadCount, runProjections: _runProjections);
}
public static ProjectionManagerNode Create(TFChunkDb db, QueuedHandler inputQueue, HttpService httpService, IPublisher[] queues)
{
var projectionManagerNode =
new ProjectionManagerNode(inputQueue, queues, db.Config.WriterCheckpoint);
httpService.SetupController(new ProjectionsController(inputQueue));
return projectionManagerNode;
}
public TFChunkReader(TFChunkDb db, ICheckpoint checkpoint)
{
Ensure.NotNull(db, "dbConfig");
Ensure.NotNull(checkpoint, "writerCheckpoint");
_db = db;
_checkpoint = checkpoint;
}
public TFChunkScavenger(TFChunkDb db, IReadIndex readIndex, long? maxChunkDataSize = null)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(readIndex, "readIndex");
_db = db;
_readIndex = readIndex;
_maxChunkDataSize = maxChunkDataSize ?? db.Config.ChunkSize;
}
public TFChunkChaser(TFChunkDb db, ICheckpoint writerCheckpoint, ICheckpoint chaserCheckpoint)
{
Ensure.NotNull(db, "dbConfig");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
Ensure.NotNull(chaserCheckpoint, "chaserCheckpoint");
_chaserCheckpoint = chaserCheckpoint;
_reader = new TFChunkReader(db, writerCheckpoint, _chaserCheckpoint.Read());
}
public TFChunkWriter(TFChunkDb db)
{
Ensure.NotNull(db, "db");
_db = db;
_writerCheckpoint = db.Config.WriterCheckpoint;
_currentChunk = db.Manager.GetChunkFor(_writerCheckpoint.Read());
if (_currentChunk == null)
throw new InvalidOperationException("No chunk given for existing position.");
}
public TFChunkReader(TFChunkDb db, ICheckpoint writerCheckpoint, long initialPosition = 0)
{
Ensure.NotNull(db, "dbConfig");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
Ensure.Nonnegative(initialPosition, "initialPosition");
_db = db;
_writerCheckpoint = writerCheckpoint;
_curPos = initialPosition;
}
public void a_null_chaser_checksum_throws_argument_null_exception()
{
var db = new TFChunkDb(new TFChunkDbConfig(Path.GetTempPath(),
new PrefixFileNamingStrategy(Path.GetTempPath(), "prefix.tf"),
10000,
0,
new InMemoryCheckpoint(),
new ICheckpoint[0]));
Assert.Throws<ArgumentNullException>(() => new TFChunkChaser(db, new InMemoryCheckpoint(), null));
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
16 * 1024,
0,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
var chunk = _db.Manager.GetChunkFor(0);
_p1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res1 = chunk.TryAppend(_p1);
_c1 = LogRecord.Commit(_res1.NewPosition, Guid.NewGuid(), _p1.LogPosition, 0);
_cres1 = chunk.TryAppend(_c1);
_p2 = LogRecord.SingleWrite(_cres1.NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res2 = chunk.TryAppend(_p2);
_c2 = LogRecord.Commit(_res2.NewPosition, Guid.NewGuid(), _p2.LogPosition, 1);
_cres2 = chunk.TryAppend(_c2);
_p3 = LogRecord.SingleWrite(_cres2.NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res3 = chunk.TryAppend(_p3);
_c3 = LogRecord.Commit(_res3.NewPosition, Guid.NewGuid(), _p3.LogPosition, 2);
_cres3 = chunk.TryAppend(_c3);
chunk.Complete();
_db.Config.WriterCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition);
_db.Config.WriterCheckpoint.Flush();
_db.Config.ChaserCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition);
_db.Config.ChaserCheckpoint.Flush();
var bus = new InMemoryBus("Bus");
var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
var scavenger = new TFChunkScavenger(_db, ioDispatcher, new FakeTableIndex(),
new FakeReadIndex(x => x == "es-to-scavenge"), Guid.NewGuid(), "fakeNodeIp");
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
_scavengedChunk = _db.Manager.GetChunk(0);
}
public void with_a_writer_checksum_of_zero_and_no_files_is_valid()
{
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
10000,
0,
new InMemoryCheckpoint(0),
new ICheckpoint[0]));
Assert.DoesNotThrow(db.OpenVerifyAndClean);
db.Dispose();
}
public TFChunkDbCreationHelper(TFChunkDbConfig dbConfig)
{
Ensure.NotNull(dbConfig, "dbConfig");
_dbConfig = dbConfig;
_db = new TFChunkDb(_dbConfig);
_db.OpenVerifyAndClean();
if (_db.Config.WriterCheckpoint.ReadNonFlushed() > 0)
throw new Exception("The DB already contains some data.");
}
public TFChunkChaser(TFChunkDb db, ICheckpoint writerCheckpoint, ICheckpoint chaserCheckpoint)
{
Ensure.NotNull(db, "dbConfig");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
Ensure.NotNull(chaserCheckpoint, "chaserCheckpoint");
_db = db;
_writerCheckpoint = writerCheckpoint;
_chaserCheckpoint = chaserCheckpoint;
_curPos = _chaserCheckpoint.Read();
}
public Projections(
TFChunkDb db, QueuedHandler mainQueue, ISubscriber mainBus, TimerService timerService, ITimeProvider timeProvider,
IHttpForwarder httpForwarder, HttpService[] httpServices, IPublisher networkSendQueue,
int projectionWorkerThreadCount, RunProjections runProjections)
{
_projectionWorkerThreadCount = projectionWorkerThreadCount;
SetupMessaging(
db, mainQueue, mainBus, timerService, timeProvider, httpForwarder, httpServices, networkSendQueue,
runProjections);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
MaxEntriesInMemTable);
var hasher = new ByLengthHasher();
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
hasher,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount);
ReadIndex.Init(ChaserCheckpoint.Read());
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
_scavenger = new TFChunkScavenger(Db, TableIndex, hasher, ReadIndex);
_scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
}
}
public override void TestFixtureTearDown()
{
using (var db = new TFChunkDb(_config))
{
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
}
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.AreEqual(1, Directory.GetFiles(PathName, "*").Length);
base.TestFixtureTearDown();
}
public void a_null_chaser_checksum_throws_argument_null_exception()
{
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Assert.Throws<ArgumentNullException>(() => new TFChunkChaser(db, new InMemoryCheckpoint(), null));
}
public TFChunkScavenger(TFChunkDb db, IODispatcher ioDispatcher, ITableIndex tableIndex, IHasher hasher, IReadIndex readIndex,
Guid scavengeId, string nodeEndpoint, long?maxChunkDataSize = null)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(ioDispatcher, "ioDispatcher");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.NotNull(hasher, "hasher");
Ensure.NotNull(nodeEndpoint, "nodeEndpoint");
Ensure.NotNull(readIndex, "readIndex");
_db = db;
_ioDispatcher = ioDispatcher;
_tableIndex = tableIndex;
_hasher = hasher;
_scavengeId = scavengeId;
_nodeEndpoint = nodeEndpoint;
_readIndex = readIndex;
_maxChunkDataSize = maxChunkDataSize ?? db.Config.ChunkSize;
}
public TFChunkScavenger(TFChunkDb db, ITFChunkScavengerLog scavengerLog, ITableIndex tableIndex, IReadIndex readIndex, long?maxChunkDataSize = null,
bool unsafeIgnoreHardDeletes = false, int threads = 1)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(scavengerLog, "scavengerLog");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.NotNull(readIndex, "readIndex");
Ensure.Positive(threads, "threads");
if (threads > MaxThreadCount)
{
Log.Warn("{numThreads} scavenging threads not allowed. Max threads allowed for scavenging is {maxThreadCount}. Capping.", threads, MaxThreadCount);
threads = MaxThreadCount;
}
_db = db;
_scavengerLog = scavengerLog;
_tableIndex = tableIndex;
_readIndex = readIndex;
_maxChunkDataSize = maxChunkDataSize ?? db.Config.ChunkSize;
_unsafeIgnoreHardDeletes = unsafeIgnoreHardDeletes;
_threads = threads;
}
