protected override void AddItemsForScenario(IMemTable memTable)
{
memTable.Add(0x0101, 0x0001, 0x0001);
memTable.Add(0x0105, 0x0001, 0x0002);
memTable.Add(0x0102, 0x0001, 0x0003);
memTable.Add(0x0102, 0x0002, 0x0004);
memTable.Add(0x0103, 0x0001, 0x0005);
}
protected override void AddItemsForScenario(IMemTable memTable)
{
memTable.Add(0x010100000000, 0, 0x0001);
memTable.Add(0x010100000000, 0, 0x0002);
memTable.Add(0x010500000000, 0, 0x0003);
memTable.Add(0x010500000000, 0, 0x0004);
memTable.Add(0x010500000000, 0, 0x0005);
}
public void Apply(IMemTable memTable)
{
IOperation operation;
while (opLogReader.Read(out operation))
{
memTable.Apply(operation.Item);
}
}
public void SetUp()
{
databaseDirectory = new DirectoryInfo("C:\\databaseLsm");
diskTablesMerger = Substitute.For <DiskTablesMerger>();
memTable = Substitute.For <IMemTable>();
method = MergeMethod.MergeByLevel;
databaseManager = new DatabaseManager(memTable, diskTablesMerger, databaseDirectory, method);
}
public DatabaseManager(IMemTable memTable, DiskTablesMerger diskTablesMerger, DirectoryInfo databaseDirectory, MergeMethod mergeMethod)
{
this.memTable = memTable;
this.diskTablesMerger = diskTablesMerger;
this.databaseDirectory = databaseDirectory;
MergeMethod = mergeMethod;
ItemsTreshold = 10;
}
public void Apply(IMemTable memTable)
{
IOperation op;
while (opLogReader.Read(out op))
{
var operation = op as Operation;
memTable.Add(operation.Item);
}
}
public static PTable FromMemtable(IMemTable table, string filename, int cacheDepth = 16)
{
if (table == null)
{
throw new ArgumentNullException("table");
}
if (filename == null)
{
throw new ArgumentNullException("filename");
}
Log.Trace("Started dumping MemTable [{0}] into PTable...", table.Id);
using (var f = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 8096,
FileOptions.SequentialScan))
{
f.SetLength(PTableHeader.Size + (table.Count << 4) + MD5Size); // EXACT SIZE
f.Seek(0, SeekOrigin.Begin);
var md5 = MD5.Create();
var buffer = new byte[16];
using (var cs = new CryptoStream(f, md5, CryptoStreamMode.Write))
using (var b = new BufferedStream(cs, 65536))
{
// WRITE HEADER
var headerBytes = new PTableHeader(Version).AsByteArray();
cs.Write(headerBytes, 0, headerBytes.Length);
// WRITE INDEX ENTRIES
foreach (var record in table.IterateAllInOrder())
{
var x = record;
AppendRecordTo(b, x.Bytes, buffer);
}
b.Flush();
cs.FlushFinalBlock();
// WRITE MD5
var hash = md5.Hash;
f.Write(hash, 0, hash.Length);
}
f.Close();
Log.Trace("Done dumping MemTable [{0}].", table.Id);
}
return(new PTable(filename, table.Id, depth: cacheDepth));
}
public static PTable FromMemtable(IMemTable table, string filename, int cacheDepth = 16)
{
Ensure.NotNull(table, "table");
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.Nonnegative(cacheDepth, "cacheDepth");
var indexEntrySize = table.Version == PTableVersions.Index32Bit ? PTable.IndexEntry32Size : PTable.IndexEntry64Size;
var sw = Stopwatch.StartNew();
using (var fs = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.None,
DefaultSequentialBufferSize, FileOptions.SequentialScan))
{
fs.SetLength(PTableHeader.Size + indexEntrySize * (long)table.Count + MD5Size); // EXACT SIZE
fs.Seek(0, SeekOrigin.Begin);
using (var md5 = MD5.Create())
using (var cs = new CryptoStream(fs, md5, CryptoStreamMode.Write))
using (var bs = new BufferedStream(cs, DefaultSequentialBufferSize))
{
// WRITE HEADER
var headerBytes = new PTableHeader(table.Version).AsByteArray();
cs.Write(headerBytes, 0, headerBytes.Length);
// WRITE INDEX ENTRIES
var buffer = new byte[indexEntrySize];
foreach (var record in table.IterateAllInOrder())
{
var rec = record;
AppendRecordTo(bs, buffer, table.Version, rec, indexEntrySize);
}
bs.Flush();
cs.FlushFinalBlock();
// WRITE MD5
var hash = md5.Hash;
fs.Write(hash, 0, hash.Length);
}
}
Log.Trace("Dumped MemTable [{0}, {1} entries] in {2}.", table.Id, table.Count, sw.Elapsed);
return(new PTable(filename, table.Id, depth: cacheDepth));
}
public static PTable FromMemtable(IMemTable table, string filename, int cacheDepth = 16)
{
Ensure.NotNull(table, "table");
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.Nonnegative(cacheDepth, "cacheDepth");
//Log.Trace("Started dumping MemTable [{0}] into PTable...", table.Id);
var sw = Stopwatch.StartNew();
using (var fs = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.None,
DefaultSequentialBufferSize, FileOptions.SequentialScan))
{
fs.SetLength(PTableHeader.Size + IndexEntrySize * (long)table.Count + MD5Size); // EXACT SIZE
fs.Seek(0, SeekOrigin.Begin);
using (var md5 = MD5.Create())
using (var cs = new CryptoStream(fs, md5, CryptoStreamMode.Write))
using (var bs = new BufferedStream(cs, DefaultSequentialBufferSize))
{
// WRITE HEADER
var headerBytes = new PTableHeader(Version).AsByteArray();
cs.Write(headerBytes, 0, headerBytes.Length);
// WRITE INDEX ENTRIES
var buffer = new byte[IndexEntrySize];
foreach (var record in table.IterateAllInOrder())
{
var rec = record;
AppendRecordTo(bs, rec.Bytes, buffer);
}
bs.Flush();
cs.FlushFinalBlock();
// WRITE MD5
var hash = md5.Hash;
fs.Write(hash, 0, hash.Length);
}
}
Log.Trace("Dumped MemTable [{0}, {1} entries] in {2}.", table.Id, table.Count, sw.Elapsed);
return new PTable(filename, table.Id, depth: cacheDepth);
}
protected abstract void AddItemsForScenario(IMemTable memTable);
public void SetUp()
{
MemTable = _memTableFactory();
}
public void SetUp()
{
MemTable = _memTableFactory();
}
public void Apply(IMemTable memTable)
{
throw new System.NotImplementedException();
}
public static PTable FromMemtable(IMemTable table, string filename, int cacheDepth = 16, bool skipIndexVerify = false)
{
Ensure.NotNull(table, "table");
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.Nonnegative(cacheDepth, "cacheDepth");
int indexEntrySize = GetIndexEntrySize(table.Version);
long dumpedEntryCount = 0;
var sw = Stopwatch.StartNew();
using (var fs = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.None,
DefaultSequentialBufferSize, FileOptions.SequentialScan))
{
var fileSize = GetFileSizeUpToIndexEntries(table.Count, table.Version);
fs.SetLength(fileSize);
fs.Seek(0, SeekOrigin.Begin);
using (var md5 = MD5.Create())
using (var cs = new CryptoStream(fs, md5, CryptoStreamMode.Write))
using (var bs = new BufferedStream(cs, DefaultSequentialBufferSize))
{
// WRITE HEADER
var headerBytes = new PTableHeader(table.Version).AsByteArray();
cs.Write(headerBytes, 0, headerBytes.Length);
// WRITE INDEX ENTRIES
var buffer = new byte[indexEntrySize];
var records = table.IterateAllInOrder();
List <Midpoint> midpoints = new List <Midpoint>();
var requiredMidpointCount = GetRequiredMidpointCountCached(table.Count, table.Version, cacheDepth);
long indexEntry = 0L;
foreach (var rec in records)
{
AppendRecordTo(bs, buffer, table.Version, rec, indexEntrySize);
dumpedEntryCount += 1;
if (table.Version >= PTableVersions.IndexV4 && IsMidpointIndex(indexEntry, table.Count, requiredMidpointCount))
{
midpoints.Add(new Midpoint(new IndexEntryKey(rec.Stream, rec.Version), indexEntry));
}
indexEntry++;
}
//WRITE MIDPOINTS
if (table.Version >= PTableVersions.IndexV4)
{
var numIndexEntries = table.Count;
if (dumpedEntryCount != numIndexEntries)
{
//if index entries have been removed, compute the midpoints again
numIndexEntries = dumpedEntryCount;
requiredMidpointCount = GetRequiredMidpointCount(numIndexEntries, table.Version, cacheDepth);
midpoints = ComputeMidpoints(bs, fs, table.Version, indexEntrySize, numIndexEntries, requiredMidpointCount, midpoints);
}
WriteMidpointsTo(bs, fs, table.Version, indexEntrySize, buffer, dumpedEntryCount, numIndexEntries, requiredMidpointCount, midpoints);
}
bs.Flush();
cs.FlushFinalBlock();
// WRITE MD5
var hash = md5.Hash;
fs.SetLength(fs.Position + MD5Size);
fs.Write(hash, 0, hash.Length);
fs.FlushToDisk();
}
}
Log.Trace("Dumped MemTable [{id}, {table} entries] in {elapsed}.", table.Id, table.Count, sw.Elapsed);
return(new PTable(filename, table.Id, depth: cacheDepth, skipIndexVerify: skipIndexVerify));
}
protected abstract void AddItemsForScenario(IMemTable memTable);
public static PTable FromMemtable(IMemTable table, string filename, int initialReaders, int maxReaders,
int cacheDepth = 16,
bool skipIndexVerify = false,
bool useBloomFilter = true,
int lruCacheSize = 1_000_000)
{
Ensure.NotNull(table, "table");
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.Nonnegative(cacheDepth, "cacheDepth");
int indexEntrySize = GetIndexEntrySize(table.Version);
long dumpedEntryCount = 0;
var sw = Stopwatch.StartNew();
using (var fs = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.None,
DefaultSequentialBufferSize, FileOptions.SequentialScan)) {
var fileSize = GetFileSizeUpToIndexEntries(table.Count, table.Version);
fs.SetLength(fileSize);
fs.Seek(0, SeekOrigin.Begin);
using (var bloomFilter = ConstructBloomFilter(useBloomFilter, filename, table.Count))
using (var md5 = MD5.Create())
using (var cs = new CryptoStream(fs, md5, CryptoStreamMode.Write))
using (var bs = new BufferedStream(cs, DefaultSequentialBufferSize)) {
// WRITE HEADER
var headerBytes = new PTableHeader(table.Version).AsByteArray();
cs.Write(headerBytes, 0, headerBytes.Length);
// WRITE INDEX ENTRIES
var buffer = new byte[indexEntrySize];
var records = table.IterateAllInOrder();
var requiredMidpointCount = GetRequiredMidpointCountCached(table.Count, table.Version, cacheDepth);
using var midpoints = new UnmanagedMemoryAppendOnlyList <Midpoint>((int)requiredMidpointCount + MidpointsOverflowSafetyNet);
long indexEntry = 0L;
ulong?previousHash = null;
foreach (var rec in records)
{
AppendRecordTo(bs, buffer, table.Version, rec, indexEntrySize);
dumpedEntryCount += 1;
if (table.Version >= PTableVersions.IndexV4 &&
IsMidpointIndex(indexEntry, table.Count, requiredMidpointCount))
{
midpoints.Add(new Midpoint(new IndexEntryKey(rec.Stream, rec.Version), indexEntry));
}
// WRITE BLOOM FILTER ENTRY
if (bloomFilter != null && rec.Stream != previousHash)
{
// we are creating a PTable of the same version as the Memtable. therefore the hash is the right format
var streamHash = rec.Stream;
bloomFilter.Add(GetSpan(ref streamHash));
previousHash = rec.Stream;
}
indexEntry++;
}
//WRITE MIDPOINTS
if (table.Version >= PTableVersions.IndexV4)
{
var numIndexEntries = table.Count;
if (dumpedEntryCount != numIndexEntries)
{
//if index entries have been removed, compute the midpoints again
numIndexEntries = dumpedEntryCount;
requiredMidpointCount =
GetRequiredMidpointCount(numIndexEntries, table.Version, cacheDepth);
ComputeMidpoints(bs, fs, table.Version, indexEntrySize, numIndexEntries,
requiredMidpointCount, midpoints);
}
WriteMidpointsTo(bs, fs, table.Version, indexEntrySize, buffer, dumpedEntryCount,
numIndexEntries, requiredMidpointCount, midpoints);
}
bloomFilter?.Flush();
bs.Flush();
cs.FlushFinalBlock();
// WRITE MD5
var hash = md5.Hash;
fs.SetLength(fs.Position + MD5Size);
fs.Write(hash, 0, hash.Length);
fs.FlushToDisk();
}
}
Log.Debug("Dumped MemTable [{id}, {table} entries] in {elapsed}.", table.Id, table.Count, sw.Elapsed);
return(new PTable(filename, table.Id, initialReaders, maxReaders, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize));
}
