public void when_checkpoints_pair_is_corrupted()
{
using (var fs = File.Open(_indexMapFileName, FileMode.Open)) {
fs.Position = 34;
var b = (byte)fs.ReadByte();
b ^= 1;
fs.Position = 34;
fs.WriteByte(b);
}
Assert.Throws <CorruptIndexException>(() =>
IndexMapTestFactory.FromFile(_indexMapFileName, maxTablesPerLevel: 2));
}
public void when_ptable_type_is_corrupted()
{
_ptable.Dispose();
_ptable = null;
using (var fs = File.Open(_ptableFileName, FileMode.Open)) {
fs.Seek(0, SeekOrigin.Begin);
fs.WriteByte(123);
}
Assert.Throws <CorruptIndexException>(() =>
IndexMapTestFactory.FromFile(_indexMapFileName, maxTablesPerLevel: 2));
}
public override async Task SetUp()
{
await base.SetUp();
_indexMapFileName = GetFilePathFor("index.map");
_ptableFileName = GetFilePathFor("ptable");
_emptyIndexMap = IndexMapTestFactory.FromFile(_indexMapFileName);
var memTable = new HashListMemTable(_ptableVersion, maxSize: 10);
memTable.Add(0, 1, 2);
_ptable = PTable.FromMemtable(memTable, _ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault);
}
public override void SetUp()
{
base.SetUp();
_indexMapFileName = GetFilePathFor("index.map");
_ptableFileName = GetFilePathFor("ptable");
_emptyIndexMap = IndexMapTestFactory.FromFile(_indexMapFileName);
var memTable = new HashListMemTable(_ptableVersion, maxSize: 10);
memTable.Add(0, 1, 2);
_ptable = PTable.FromMemtable(memTable, _ptableFileName);
}
public void when_ptable_hash_is_corrupted()
{
_ptable.Dispose();
_ptable = null;
using (var fs = File.Open(_ptableFileName, FileMode.Open)) {
fs.Seek(-PTable.MD5Size, SeekOrigin.End);
var b = (byte)fs.ReadByte();
b ^= 1;
fs.Seek(-PTable.MD5Size, SeekOrigin.End);
fs.WriteByte(b);
}
Assert.Throws <CorruptIndexException>(() =>
IndexMapTestFactory.FromFile(_indexMapFileName, maxTablesPerLevel: 2));
}
public void when_ptable_data_is_corrupted()
{
_ptable.Dispose();
_ptable = null;
using (var fs = File.Open(_ptableFileName, FileMode.Open)) {
fs.Position = new Random().Next(PTableHeader.Size, (int)fs.Length);
var b = (byte)fs.ReadByte();
b ^= 1;
fs.Position -= 1;
fs.WriteByte(b);
}
Assert.Throws <CorruptIndexException>(() =>
IndexMapTestFactory.FromFile(_indexMapFileName, maxTablesPerLevel: 2));
}
public void should_merge_levels_above_max_level()
{
AddTables(201); //gives 1 level 0, 1 level 3 and 6 level 5s
Assert.AreEqual(8, _result.MergedMap.InOrder().Count());
var filename = GetFilePathFor("changemaxlevel");
_result.MergedMap.SaveToFile(filename);
_result.MergedMap.Dispose(TimeSpan.FromMilliseconds(100));
_map.Dispose(TimeSpan.FromMilliseconds(100));
_map = IndexMapTestFactory.FromFile(filename, maxAutoMergeLevel: 3);
_result = _map.TryManualMerge(
UpgradeHash, ExistsAt,
RecordExistsAt, _fileNameProvider, _ptableVersion,
skipIndexVerify: _skipIndexVerify);
Assert.AreEqual(2, _result.MergedMap.InOrder().Count());
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_mergeFile = GetTempFilePath();
_filename = GetTempFilePath();
_map = IndexMapTestFactory.FromFile(_filename, maxTablesPerLevel: 4);
var memtable = new HashListMemTable(_ptableVersion, maxSize: 10);
memtable.Add(0, 1, 0);
_result = _map.AddPTable(
PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 1, 2,
(streamId, hash) => hash,
_ => true,
_ => new System.Tuple <string, bool>("", true),
new GuidFilenameProvider(PathName),
_ptableVersion,
_maxAutoMergeIndexLevel,
0,
skipIndexVerify: _skipIndexVerify);
_result.ToDelete.ForEach(x => x.MarkForDestruction());
_result = _result.MergedMap.AddPTable(
PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 3, 4,
(streamId, hash) => hash,
_ => true, _ => new System.Tuple <string, bool>("", true), new GuidFilenameProvider(PathName),
_ptableVersion, _maxAutoMergeIndexLevel, 0, skipIndexVerify: _skipIndexVerify);
_result.ToDelete.ForEach(x => x.MarkForDestruction());
_result = _result.MergedMap.AddPTable(
PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 4, 5,
(streamId, hash) => hash,
_ => true, _ => new System.Tuple <string, bool>("", true), new GuidFilenameProvider(PathName),
_ptableVersion, _maxAutoMergeIndexLevel, 0, skipIndexVerify: _skipIndexVerify);
_result.ToDelete.ForEach(x => x.MarkForDestruction());
_result = _result.MergedMap.AddPTable(
PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 1,
(streamId, hash) => hash,
_ => true, _ => new System.Tuple <string, bool>("", true), new FakeFilenameProvider(_mergeFile),
_ptableVersion, _maxAutoMergeIndexLevel, 0, skipIndexVerify: _skipIndexVerify);
_result.ToDelete.ForEach(x => x.MarkForDestruction());
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_filename = GetTempFilePath();
_tablename = GetTempFilePath();
_mergeFile = GetFilePathFor("mergefile");
_map = IndexMapTestFactory.FromFile(_filename, maxAutoMergeLevel: _maxAutoMergeIndexLevel);
var memtable = new HashListMemTable(_ptableVersion, maxSize: 10);
memtable.Add(0, 1, 0);
var table = PTable.FromMemtable(memtable, _tablename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify);
_result = _map.AddAndMergePTable(table, 7, 11, (streamId, hash) => hash, _ => true,
_ => new System.Tuple <string, bool>("", true), new FakeFilenameProvider(_mergeFile), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify);
table.MarkForDestruction();
}
public override async Task SetUp()
{
await base.SetUp();
_indexMapFileName = GetFilePathFor("index.map");
_ptableFileName = GetFilePathFor("ptable");
var indexMap = IndexMapTestFactory.FromFile(_indexMapFileName, maxTablesPerLevel: 2, maxAutoMergeLevel: _maxAutoMergeIndexLevel);
var memtable = new HashListMemTable(_ptableVersion, maxSize: 10);
memtable.Add(0, 0, 0);
memtable.Add(1, 1, 100);
_ptable = PTable.FromMemtable(memtable, _ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify);
indexMap = indexMap.AddAndMergePTable(_ptable, 0, 0, (streamId, hash) => hash, _ => true,
_ => new Tuple <string, bool>("", true), new GuidFilenameProvider(PathName), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify).MergedMap;
indexMap.SaveToFile(_indexMapFileName);
}
public override void SetUp()
{
base.SetUp();
_indexMapFileName = GetFilePathFor("index.map");
_ptableFileName = GetFilePathFor("ptable");
var indexMap = IndexMapTestFactory.FromFile(_indexMapFileName, maxTablesPerLevel: 2);
var memtable = new HashListMemTable(_ptableVersion, maxSize: 10);
memtable.Add(0, 0, 0);
memtable.Add(1, 1, 100);
_ptable = PTable.FromMemtable(memtable, _ptableFileName, skipIndexVerify: _skipIndexVerify);
indexMap = indexMap.AddPTable(_ptable, 0, 0, (streamId, hash) => hash, _ => true,
_ => new Tuple <string, bool>("", true), new GuidFilenameProvider(PathName), _ptableVersion,
_maxAutoMergeIndexLevel, 0, skipIndexVerify: _skipIndexVerify).MergedMap;
indexMap.SaveToFile(_indexMapFileName);
}
public override void SetUp()
{
base.SetUp();
_filename = GetFilePathFor("indexfile");
_tablename = GetTempFilePath();
_mergeFile = GetFilePathFor("outfile");
_map = IndexMapTestFactory.FromFile(_filename, maxTablesPerLevel: 4);
var memtable = new HashListMemTable(_ptableVersion, maxSize: 10);
memtable.Add(0, 2, 123);
var table = PTable.FromMemtable(memtable, _tablename);
_result = _map.AddPTable(table, 0, 0, (streamId, hash) => hash, _ => true,
_ => new Tuple <string, bool>("", true), new FakeFilenameProvider(_mergeFile), _ptableVersion,
_maxAutoMergeIndexLevel, 0);
_result = _result.MergedMap.AddPTable(table, 0, 0, (streamId, hash) => hash, _ => true,
_ => new Tuple <string, bool>("", true), new FakeFilenameProvider(_mergeFile), _ptableVersion,
_maxAutoMergeIndexLevel, 0);
_result = _result.MergedMap.AddPTable(table, 0, 0, (streamId, hash) => hash, _ => true,
_ => new Tuple <string, bool>("", true), new FakeFilenameProvider(_mergeFile), _ptableVersion,
_maxAutoMergeIndexLevel, 0);
var merged = _result.MergedMap.AddPTable(table, 0, 0, (streamId, hash) => hash, _ => true,
_ => new Tuple <string, bool>("", true), new FakeFilenameProvider(_mergeFile), _ptableVersion,
_maxAutoMergeIndexLevel, 0);
_result = merged.MergedMap.AddPTable(table, 0, 0, (streamId, hash) => hash, _ => true,
_ => new Tuple <string, bool>("", true), new FakeFilenameProvider(_mergeFile), _ptableVersion,
_maxAutoMergeIndexLevel, 0);
_result = _result.MergedMap.AddPTable(table, 7, 11, (streamId, hash) => hash, _ => true,
_ => new Tuple <string, bool>("", true), new FakeFilenameProvider(_mergeFile), _ptableVersion,
_maxAutoMergeIndexLevel, 0);
_result.MergedMap.SaveToFile(_filename);
table.Dispose();
merged.MergedMap.InOrder().ToList().ForEach(x => x.Dispose());
merged.ToDelete.ForEach(x => x.Dispose());
_result.MergedMap.InOrder().ToList().ForEach(x => x.Dispose());
_result.ToDelete.ForEach(x => x.Dispose());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_filename = GetTempFilePath();
_tablename = GetTempFilePath();
_mergeFile = GetFilePathFor("mergefile");
_map = IndexMapTestFactory.FromFile(_filename);
var memtable = new HashListMemTable(_ptableVersion, maxSize: 10);
memtable.Add(0, 1, 0);
var table = PTable.FromMemtable(memtable, _tablename, skipIndexVerify: _skipIndexVerify);
_result = _map.AddPTable(table, 7, 11, (streamId, hash) => hash, _ => true,
_ => new System.Tuple <string, bool>("", true), new FakeFilenameProvider(_mergeFile), _ptableVersion,
_maxAutoMergeIndexLevel, 0, skipIndexVerify: _skipIndexVerify);
table.MarkForDestruction();
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_filename = GetFilePathFor("indexfile");
_tablename = GetTempFilePath();
_mergeFile = GetFilePathFor("outputfile");
_map = IndexMapTestFactory.FromFile(_filename, maxAutoMergeLevel: _maxAutoMergeIndexLevel);
var memtable = new HashListMemTable(_ptableVersion, maxSize: 10);
memtable.Add(0, 2, 7);
var table = PTable.FromMemtable(memtable, _tablename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault);
_result = _map.AddAndMergePTable(table, 7, 11, (streamId, hash) => hash, _ => true,
_ => new Tuple <string, bool>("", true), new FakeFilenameProvider(_mergeFile), _ptableVersion, 0);
_result.MergedMap.SaveToFile(_filename);
_result.ToDelete.ForEach(x => x.Dispose());
_result.MergedMap.InOrder().ToList().ForEach(x => x.Dispose());
table.Dispose();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_mergeFile = GetTempFilePath();
_filename = GetTempFilePath();
_map = IndexMapTestFactory.FromFile(_filename, maxTablesPerLevel: 2);
var memtable = new HashListMemTable(_ptableVersion, maxSize: 10);
memtable.Add(0, 1, 0);
_result = _map.AddPTable(
PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify),
10, 20, (streamId, hash) => hash, _ => true, _ => new Tuple <string, bool>("", true),
new GuidFilenameProvider(PathName), _ptableVersion, _maxAutoMergeIndexLevel, 0,
skipIndexVerify: _skipIndexVerify);
_result.ToDelete.ForEach(x => x.MarkForDestruction());
_result = _result.MergedMap.AddPTable(
PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify),
20, 30, (streamId, hash) => hash, _ => true, _ => new Tuple <string, bool>("", true),
new GuidFilenameProvider(PathName), _ptableVersion, _maxAutoMergeIndexLevel, 0,
skipIndexVerify: _skipIndexVerify);
_result.ToDelete.ForEach(x => x.MarkForDestruction());
_result = _result.MergedMap.AddPTable(
PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify),
30, 40, (streamId, hash) => hash, _ => true, _ => new Tuple <string, bool>("", true),
new GuidFilenameProvider(PathName), _ptableVersion, _maxAutoMergeIndexLevel, 0,
skipIndexVerify: _skipIndexVerify);
_result.ToDelete.ForEach(x => x.MarkForDestruction());
_result = _result.MergedMap.AddPTable(
PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify),
50, 60, (streamId, hash) => hash, _ => true, _ => new Tuple <string, bool>("", true),
new FakeFilenameProvider(_mergeFile + ".firstmerge", _mergeFile), _ptableVersion,
_maxAutoMergeIndexLevel, 0, skipIndexVerify: _skipIndexVerify);
_result.ToDelete.ForEach(x => x.MarkForDestruction());
}
public void should_open_if_auto_merge_levels_match()
{
Assert.DoesNotThrow(() => IndexMapTestFactory.FromFile(_filename, maxAutoMergeLevel: 4));
}
public void should_throw_if_max_auto_merge_is_larger_than_map_value()
{
Assert.Throws <CorruptIndexException>(() => IndexMapTestFactory.FromFile(_filename, maxAutoMergeLevel: 5));
}
public virtual void Setup()
{
_filename = GetTempFilePath();
_fileNameProvider = new GuidFilenameProvider(PathName);
_map = IndexMapTestFactory.FromFile(_filename, maxTablesPerLevel: 2, maxAutoMergeLevel: _maxAutoMergeLevel);
}
public void throw_corruptedindexexception_when_commit_checkpoint_is_less_than_minus_one()
{
CreateArtificialIndexMapFile(_indexMapFileName, 0, -2, null);
Assert.Throws <CorruptIndexException>(() =>
IndexMapTestFactory.FromFile(_indexMapFileName, maxTablesPerLevel: 2));
}
throw_corruptedindexexception_if_prepare_checkpoint_is_minus_one_and_there_are_ptables_in_indexmap()
{
CreateArtificialIndexMapFile(_indexMapFileName, -1, 0, _ptableFileName);
Assert.Throws <CorruptIndexException>(() =>
IndexMapTestFactory.FromFile(_indexMapFileName, maxTablesPerLevel: 2));
}
public void Setup()
{
_map = IndexMapTestFactory.FromFile("thisfiledoesnotexist");
}
public void ConstructTableIndexWithCorruptIndexEntries(byte version, bool skipIndexVerify,
bool createForceVerifyFile = false)
{
base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var fakeReader = new TFReaderLease(new FakeIndexReader());
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
int.MaxValue,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
//create index entries
for (int i = 1; i <= NumIndexEntries; i++)
{
_tableIndex.Add(i * 1337, StreamName, i, i * 1337);
}
_tableIndex.Close(false);
//load index map to obtain ptable filenames
_indexMap = IndexMapTestFactory.FromFile(Path.Combine(PathName, TableIndex.IndexMapFilename));
List <string> ptableFiles = new List <string>();
foreach (string ptableFilename in _indexMap.GetAllFilenames())
{
ptableFiles.Add(ptableFilename);
}
_indexMap.Dispose(TimeSpan.FromSeconds(5));
//corrupt ptable files
foreach (string ptableFilename in ptableFiles)
{
CorruptPTableFile(ptableFilename, version, "zeroOutMiddleEntry");
}
//create force verify file if requested
if (createForceVerifyFile)
{
using (FileStream fs = new FileStream(Path.Combine(PathName, TableIndex.ForceIndexVerifyFilename),
FileMode.OpenOrCreate)) {
}
;
}
//load table index again
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
int.MaxValue,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify,
indexCacheDepth: 8);
_tableIndex.Initialize(long.MaxValue);
}