public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV1,
5,
maxSizeForMemory: 5 + _extraStreamHashesAtBeginning + _extraStreamHashesAtEnd,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
Assert.Greater(_lowHasher.Hash("abcd"), _lowHasher.Hash("LPN-FC002_LPK51001"));
for (int i = 0; i < _extraStreamHashesAtBeginning; i++)
{
_tableIndex.Add(1, "abcd", i, i + 1);
}
Assert.Less(_lowHasher.Hash("wxyz"), _lowHasher.Hash("LPN-FC002_LPK51001"));
for (int i = 0; i < _extraStreamHashesAtEnd; i++)
{
_tableIndex.Add(1, "wxyz", i, i + 1);
}
_tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
_tableIndex.Add(1, "account--696193173", 0, 2);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
_tableIndex.Add(1, "account--696193173", 1, 4);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
5,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "account--696193173", 2, 6);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
_tableIndex.Add(1, "account--696193173", 3, 8);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
_tableIndex.Add(1, "account--696193173", 4, 10);
Thread.Sleep(500);
}
protected override void when()
{
if (_useMaxAge)
{
_indexBackend.SetStreamMetadata(streamId, new StreamMetadata(maxAge: TimeSpan.FromDays(1)));
}
//ptable 1 with 32bit indexes
_tableIndex.Add(1, streamId, 0, 2);
_tableIndex.Add(1, streamId, 1, 4);
_tableIndex.Add(1, streamId, 2, 6);
System.Threading.Thread.Sleep(500);
_tableIndex.Close(false);
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV2, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.IndexV2,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader <string>(
_indexBackend, _tableIndex,
_logFormat.StreamNamesProvider,
_logFormat.StreamIdValidator,
_logFormat.StreamExistenceFilterReader,
new EventStore.Core.Data.StreamMetadata(),
_hashCollisionReadLimit, skipIndexScanOnRead: false);
//memtable with 64bit indexes
_tableIndex.Add(1, streamId, 0, 8);
}
public void ConstructTableIndexWithCorruptIndexEntries(byte version, bool skipIndexVerify, bool createForceVerifyFile = false)
{
base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var fakeReader = new TFReaderLease(new FakeIndexReader());
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
5,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
//create index entries
for (int i = 1; i <= NumIndexEntries; i++)
{
_tableIndex.Add(i * 1337, StreamName, i, i * 1337);
}
_tableIndex.Close(false);
//load index map to obtain ptable filenames
_indexMap = IndexMap.FromFile(Path.Combine(PathName, TableIndex.IndexMapFilename));
List <string> ptableFiles = new List <string>();
foreach (string ptableFilename in _indexMap.GetAllFilenames())
{
ptableFiles.Add(ptableFilename);
}
_indexMap.Dispose(TimeSpan.FromSeconds(5));
//corrupt ptable files
foreach (string ptableFilename in ptableFiles)
{
CorruptPTableFile(ptableFilename, version, "zeroOutMiddleEntry");
}
//create force verify file if requested
if (createForceVerifyFile)
{
using (FileStream fs = new FileStream(Path.Combine(PathName, TableIndex.ForceIndexVerifyFilename), FileMode.OpenOrCreate)){
};
}
//load table index again
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
5,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify,
indexCacheDepth: 8);
_tableIndex.Initialize(long.MaxValue);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV2, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV2,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 1);
_tableIndex.Add(1, "testStream-2", 0, 2);
_tableIndex.Add(1, "testStream-1", 1, 3);
_tableIndex.Add(1, "testStream-2", 1, 4);
_tableIndex.Add(1, "testStream-1", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-2", 2, 6);
_tableIndex.Add(1, "testStream-1", 3, 7);
_tableIndex.Add(1, "testStream-2", 3, 8);
_tableIndex.Add(1, "testStream-1", 4, 9);
_tableIndex.Add(1, "testStream-2", 4, 10);
await Task.Delay(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV1,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
_tableIndex.Add(1, "account--696193173", 0, 2);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
_tableIndex.Add(1, "account--696193173", 1, 4);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "account--696193173", 2, 6);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
_tableIndex.Add(1, "account--696193173", 3, 8);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
_tableIndex.Add(1, "account--696193173", 4, 10);
Thread.Sleep(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader2());
_lowHasher = new ByLengthHasher();
_highHasher = new ByLengthHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 3),
() => fakeReader,
PTableVersions.IndexV1,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream1, 0, 1);
_tableIndex.Add(1, Stream2, 0, 2);
_tableIndex.Add(1, Stream3, 0, 3);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 3),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream3, 1, 4);
_tableIndex.Add(1, Stream2, 1, 5);
_tableIndex.Add(1, Stream1, 1, 6);
await Task.Delay(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader2());
_lowHasher = new ByLengthHasher();
_highHasher = new ByLengthHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 3),
() => fakeReader,
PTableVersions.IndexV1,
5,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream1, 0, 1);
_tableIndex.Add(1, Stream2, 0, 2);
_tableIndex.Add(1, Stream3, 0, 3);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 3),
() => fakeReader,
_ptableVersion,
5,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream3, 1, 4);
_tableIndex.Add(1, Stream2, 1, 5);
_tableIndex.Add(1, Stream1, 1, 6);
Thread.Sleep(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
int readerCount = 0;
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => {
readerCount++;
if (readerCount < 4) // One for each table add.
{
return(fakeReader);
}
throw new Exception("Expected exception");
},
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, CancellationToken.None),
Throws.Exception.With.Message.EqualTo("Expected exception"));
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
protected override void when()
{
//ptable 1 with 32bit indexes
_tableIndex.Add(1, streamId, 0, 2);
_tableIndex.Add(1, streamId, 1, 4);
_tableIndex.Add(1, streamId, 2, 6);
System.Threading.Thread.Sleep(500);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.Index64Bit,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit);
//memtable with 64bit indexes
_tableIndex.Add(1, streamId, 0, 8);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var cancellationTokenSource = new CancellationTokenSource();
var fakeReader = new TFReaderLease(new FakeIndexReader(l => {
cancellationTokenSource.Cancel();
return(true);
}));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, cancellationTokenSource.Token),
Throws.InstanceOf <OperationCanceledException>());
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
private void AddEventToSut(string stream, int eventNumber)
{
var record = LogRecord.SingleWrite(
factory: new LogV2RecordFactory(),
logPosition: _nextLogPosition,
correlationId: Guid.NewGuid(),
eventId: Guid.NewGuid(),
eventStreamId: stream,
expectedVersion: eventNumber - 1,
eventType: "eventType",
data: ReadOnlyMemory <byte> .Empty,
metadata: ReadOnlyMemory <byte> .Empty);
_log.AddRecord(record, _nextLogPosition);
_tableIndex.Add(
commitPos: _nextLogPosition,
streamId: stream,
version: record.ExpectedVersion + 1,
position: _nextLogPosition);
_nextLogPosition += _recordOffset;
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader(l => !Deleted.Contains(l)));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5, skipIndexVerify: _skipIndexVerify,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
_tableIndex.Scavenge(_log, CancellationToken.None);
// Check it's loadable.
_tableIndex.Close(false);
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => throw new Exception("Expected exception") /* throw an exception when the first PTable scavenge starts and tries to acquire a reader */,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, CancellationToken.None),
Throws.Exception.With.Message.EqualTo("Expected exception"));
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_tableIndex = new TableIndex(_indexDir,
() => new HashListMemTable(maxSize: 10),
() => fakeReader,
maxSizeForMemory: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // in memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // in memtable
}
public void throw_argumentoutofrangeexception_on_adding_entry_with_negative_commit_position()
{
Assert.Throws <ArgumentOutOfRangeException>(() => _tableIndex.Add(-1, 0x0000, 0, 0));
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(PathName, _lowHasher, _highHasher,
() => new HashListMemTable(version: _ptableVersion, maxSize: 40),
() => { throw new InvalidOperationException(); },
_ptableVersion,
maxSizeForMemory: 20);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF00);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF01);
_tableIndex.Add(0, "0xJEEP", 0, 0xFF00);
_tableIndex.Add(0, "0xJEEP", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 0, 0xFF00);
_tableIndex.Add(0, "0xABBA", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 2, 0xFF02);
_tableIndex.Add(0, "0xABBA", 3, 0xFF03);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF10);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF11);
_tableIndex.Add(0, "0xADA", 0, 0xFF00);
}
public override void SetUp()
{
base.SetUp();
_indexDir = base.PathName;
_tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), maxSizeForMemory: 2000);
_tableIndex.Initialize();
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = base.PathName;
_tableIndex = new TableIndex(_indexDir,
() => new HashListMemTable(maxSize: 10),
maxSizeForMemory: 2,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF02);
_tableIndex.Add(0, 0xABBA, 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
_tableIndex.Add(0, 0xCEED, 10, 0xFFF1);
_tableIndex.Add(0, 0xBABA, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, 0xBABA, 1, 0xFF01);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
// memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF01);
Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
5,
maxSizeForMemory: 5,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF00);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF01);
_tableIndex.Add(0, "0xBEEF", 0, 0xFF00);
_tableIndex.Add(0, "0xBEEF", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0, "0xABBA", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 2, 0xFF02);
_tableIndex.Add(0, "0xABBA", 3, 0xFF03);
_tableIndex.Add(0, "0xADA", 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0, "0xDEAD", 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0, "0xDEAD", 1, 0xFF11); // in memtable
_tableIndex.Add(0, "0xADA", 0, 0xFF00); // in memtable
}
public override void SetUp()
{
base.SetUp();
_indexDir = base.PathName;
_tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), maxSizeForMemory: 5);
_tableIndex.Initialize();
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // in memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // in memtable
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new FakeIndexHasher();
_highHasher = new FakeIndexHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
maxSizeForMemory: 2,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, "1", 0, 0xFF00);
_tableIndex.Add(0, "1", 1, 0xFF01);
_tableIndex.Add(0, "2", 0, 0xFF00);
_tableIndex.Add(0, "2", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF00);
_tableIndex.Add(0, "3", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF02);
_tableIndex.Add(0, "3", 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, "4", 0, 0xFF00);
_tableIndex.Add(0, "5", 10, 0xFFF1);
_tableIndex.Add(0, "6", 0, 0xFF00);
_tableIndex.Add(0, "1", 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, "6", 1, 0xFF01);
_tableIndex.Add(0, "1", 1, 0xFF11);
// memtable
_tableIndex.Add(0, "4", 0, 0xFF01);
Thread.Sleep(500);
}
public void SetUp()
{
_indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
_tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 2000);
_tableIndex.Initialize();
_tableIndex.Add(0xDEAD, 0, 0xFF00);
_tableIndex.Add(0xDEAD, 1, 0xFF01);
_tableIndex.Add(0xBEEF, 0, 0xFF00);
_tableIndex.Add(0xBEEF, 1, 0xFF01);
_tableIndex.Add(0xABBA, 0, 0xFF00);
_tableIndex.Add(0xABBA, 1, 0xFF01);
_tableIndex.Add(0xABBA, 2, 0xFF02);
_tableIndex.Add(0xABBA, 3, 0xFF03);
_tableIndex.Add(0xDEAD, 0, 0xFF10);
_tableIndex.Add(0xDEAD, 1, 0xFF11);
_tableIndex.Add(0xADA, 0, 0xFF00);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new FakeIndexHasher();
_highHasher = new FakeIndexHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 2,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, "1", 0, 0xFF00);
_tableIndex.Add(0, "1", 1, 0xFF01);
_tableIndex.Add(0, "2", 0, 0xFF00);
_tableIndex.Add(0, "2", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF00);
_tableIndex.Add(0, "3", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF02);
_tableIndex.Add(0, "3", 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, "4", 0, 0xFF00);
_tableIndex.Add(0, "5", 10, 0xFFF1);
_tableIndex.Add(0, "6", 0, 0xFF00);
_tableIndex.Add(0, "1", 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, "6", 1, 0xFF01);
_tableIndex.Add(0, "1", 1, 0xFF11);
// memtable
_tableIndex.Add(0, "4", 0, 0xFF01);
await Task.Delay(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(PathName, _lowHasher, _highHasher, "",
() => new HashListMemTable(version: _ptableVersion, maxSize: 40),
() => { throw new InvalidOperationException(); },
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 5,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF00);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF01);
_tableIndex.Add(0, "0xJEEP", 0, 0xFF00);
_tableIndex.Add(0, "0xJEEP", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 0, 0xFF00);
_tableIndex.Add(0, "0xABBA", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 2, 0xFF02);
_tableIndex.Add(0, "0xABBA", 3, 0xFF03);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF10);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF11);
_tableIndex.Add(0, "0xADA", 0, 0xFF00);
_tableIndex.Add(0, "0xJEEP", 2, 0xFFF0);
_tableIndex.Add(0, "0xJEEP", 3, 0xFFF1);
}
public void SetUp()
{
_indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
_tableIndex = new TableIndex(_indexDir,
() => new HashListMemTable(),
maxSizeForMemory: 2,
maxTablesPerLevel: 2);
_tableIndex.Initialize();
// ptable level 2
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF02);
_tableIndex.Add(0, 0xABBA, 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
_tableIndex.Add(0, 0xCEED, 10, 0xFFF1);
_tableIndex.Add(0, 0xBABA, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, 0xBABA, 1, 0xFF01);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
// memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF01);
Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var scavengeBlocker = new ManualResetEventSlim(false);
var scavengeStarted = new ManualResetEventSlim(false);
var fakeReader = new TFReaderLease(new FakeIndexReader(l =>
{
scavengeStarted.Set();
if (!scavengeBlocker.Wait(5000))
{
throw new Exception("Failed to continue.");
}
return(false);
}));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.WaitForBackgroundTasks();
_log = new FakeTFScavengerLog();
var task = Task.Run(() => _tableIndex.Scavenge(_log, CancellationToken.None));
Assert.That(scavengeStarted.Wait(5000));
// Add enough for 2 more tables
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
// Release the scavenge process
scavengeBlocker.Set();
task.Wait();
// Check it's loadable.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
public void SetUp()
{
_indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
_tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 5);
_tableIndex.Initialize();
_tableIndex.Add(0xDEAD, 0, 0xFF00);
_tableIndex.Add(0xDEAD, 1, 0xFF01);
_tableIndex.Add(0xBEEF, 0, 0xFF00);
_tableIndex.Add(0xBEEF, 1, 0xFF01);
_tableIndex.Add(0xABBA, 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0xABBA, 1, 0xFF01);
_tableIndex.Add(0xABBA, 2, 0xFF02);
_tableIndex.Add(0xABBA, 3, 0xFF03);
_tableIndex.Add(0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0xDEAD, 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0xDEAD, 1, 0xFF11); // in memtable
_tableIndex.Add(0xADA, 0, 0xFF00); // in memtable
}
