internal void Apply(MemTable memTable, Reference<ulong> seq)
{
foreach (var operation in _operations)
{
var itemType = operation.Op == Operations.Delete ? ItemType.Deletion : ItemType.Value;
memTable.Add(seq.Value++, itemType, operation.Key, operation.Handle);
}
}
public void T1()
{
var storageState = new StorageState("test", new StorageOptions());
var random = new Random();
var tables = new List<MemTable>();
var expectedCount = 0;
try
{
ulong seq = 0;
for (var i = 0; i < 100; i++)
{
var table = new MemTable(storageState);
for (var j = 0; j < 1000; j++)
{
var k = random.Next();
var key = string.Format("{0:0000000000000000}", k);
table.Add(seq++, ItemType.Value, key, null);
expectedCount++;
}
tables.Add(table);
}
var iterators = tables.Select(table => table.NewIterator()).ToList();
var comparator = new InternalKeyComparator(new CaseInsensitiveComparator());
using (var iterator = new MergingIterator(comparator, iterators))
{
var actualCount = 0;
iterator.SeekToFirst();
Assert.True(iterator.IsValid);
Slice prev = string.Empty;
while (iterator.IsValid)
{
if (!prev.IsEmpty())
{
Assert.True(comparator.Compare(iterator.Key, prev) > 0);
}
prev = iterator.Key.Clone();
iterator.Next();
actualCount++;
}
Assert.Equal(expectedCount, actualCount);
}
}
finally
{
foreach (var table in tables)
table.Dispose();
}
}
public void WillNotShowValueFromLaterSnapshot()
{
var storageState = new StorageState("test", new StorageOptions());
using (var memtable = new MemTable(storageState))
{
memtable.Add(2, ItemType.Value, "test", memtable.Write(new MemoryStream(new byte[] { 1, 2, 3 })));
Stream stream;
Assert.False(memtable.TryGet("test", 1, out stream));
}
}
public void Empty()
{
var storageState = new StorageState("test", new StorageOptions());
using (var memtable = new MemTable(storageState))
{
Stream stream;
Assert.False(memtable.TryGet("test", 1, out stream));
Assert.Null(stream);
}
}
public void DeletesWillHideValues()
{
var storageState = new StorageState("test", new StorageOptions());
using (var memtable = new MemTable(storageState))
{
memtable.Add(2, ItemType.Value, "test", memtable.Write(new MemoryStream(new byte[] { 1, 2, 3 })));
memtable.Add(3, ItemType.Deletion, "test", null);
Stream stream;
Assert.True(memtable.TryGet("test", 5, out stream));
Assert.Null(stream);
}
}
public StorageState(string name, StorageOptions options)
{
_perfCounters = new PerfCounters(name);
_cancellationTokenSource = new CancellationTokenSource();
CancellationToken = _cancellationTokenSource.Token;
Options = options;
InternalKeyComparator = new InternalKeyComparator(options.Comparator);
DatabaseName = name;
Lock = new AsyncLock();
FileSystem = new FileSystem(DatabaseName);
MemTable = new MemTable(this);
TableCache = new TableCache(this);
VersionSet = new VersionSet(this);
Compactor = new BackgroundCompactor(this);
Snapshooter = new Snapshooter(this);
}
public void CanAddAndGetUsingLaterSnapshot()
{
var storageState = new StorageState("test", new StorageOptions());
using (var memtable = new MemTable(storageState))
{
memtable.Add(1, ItemType.Value, "test", memtable.Write(new MemoryStream(new byte[] { 1, 2, 3 })));
Stream stream;
Assert.True(memtable.TryGet("test", 2, out stream));
using (stream)
{
Assert.Equal(1, stream.ReadByte());
Assert.Equal(2, stream.ReadByte());
Assert.Equal(3, stream.ReadByte());
}
}
}
public void Remove(MemTable memTable, Reference<ulong> seq)
{
foreach (var operation in _operations)
{
var itemType = operation.Op == Operations.Delete ? ItemType.Deletion : ItemType.Value;
memTable.Remove(seq.Value++, itemType, operation.Key);
}
}
internal void Prepare(MemTable memTable)
{
foreach (var operation in _operations)
{
operation.Handle = memTable.Write(operation.Value);
if (DontDisposeStreamsAfterWrite)
continue;
operation.Value.Dispose();
}
}
private static IEnumerable<RavenJObject> YieldMemTableContents(MemTable memTable, Reference<Slice> target)
{
using (var iterator = memTable.NewIterator())
{
iterator.Seek(target.Value);
while (iterator.IsValid)
{
target.Value = iterator.Key;
using (var stream = iterator.CreateValueStream())
{
yield return RavenJObject.Load(new BsonReader(stream));
}
iterator.Next();
}
}
}
public void WriteLevel0Table(MemTable memTable, Version currentVersion, VersionEdit edit)
{
var stopwatch = Stopwatch.StartNew();
var fileNumber = state.VersionSet.NewFileNumber();
pendingOutputs.Add(fileNumber);
var fileMetadata = state.BuildTable(memTable, fileNumber);
pendingOutputs.Remove(fileNumber);
// Note that if file_size is zero, the file has been deleted and
// should not be added to the manifest.
int level = 0;
if (fileMetadata.FileSize > 0)
{
var minUserKey = fileMetadata.SmallestKey.UserKey;
var maxUserKey = fileMetadata.LargestKey.UserKey;
if (currentVersion != null)
{
level = currentVersion.PickLevelForMemTableOutput(minUserKey, maxUserKey);
}
edit.AddFile(level, fileMetadata);
}
state.CompactionStats[level].Add(new CompactionStats
{
Milliseconds = stopwatch.ElapsedMilliseconds,
BytesRead = 0,
BytesWritten = fileMetadata.FileSize
});
}
public MemoryIterator(MemTable table, SkipList<InternalKey, UnamangedMemoryAccessor.MemoryHandle>.Iterator iterator)
{
_table = table;
_iterator = iterator;
}
internal async Task MakeRoomForWriteAsync(bool force, AsyncLock.LockScope lockScope)
{
bool allowDelay = force == false;
while (true)
{
using (await lockScope.LockAsync())
{
if (BackgroundTask.IsCanceled || BackgroundTask.IsFaulted)
{
BackgroundTask.Wait(); // throws
}
else if (allowDelay && VersionSet.GetNumberOfFilesAtLevel(0) >= Config.SlowdownWritesTrigger)
{
// We are getting close to hitting a hard limit on the number of
// L0 files. Rather than delaying a single write by several
// seconds when we hit the hard limit, start delaying each
// individual write by 1ms to reduce latency variance. Also,
// this delay hands over some CPU to the compaction thread in
// case it is sharing the same core as the writer.
lockScope.Exit();
{
await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false);
}
await lockScope.LockAsync().ConfigureAwait(false);
allowDelay = false; // Do not delay a single write more than once
}
else if (force == false && MemTable.ApproximateMemoryUsage <= Options.WriteBatchSize)
{
// There is room in current memtable
break;
}
else if (ImmutableMemTable != null)
{
Compactor.MaybeScheduleCompaction(lockScope);
lockScope.Exit();
// We have filled up the current memtable, but the previous
// one is still being compacted, so we wait.
await BackgroundTask.ConfigureAwait(false);
}
else if (VersionSet.GetNumberOfFilesAtLevel(0) >= Config.StopWritesTrigger)
{
Compactor.MaybeScheduleCompaction(lockScope);
lockScope.Exit();
// There are too many level-0 files.
await BackgroundTask.ConfigureAwait(false);
}
else
{
// Attempt to switch to a new memtable and trigger compaction of old
Debug.Assert(VersionSet.PrevLogNumber == 0);
LogWriter.Dispose();
CreateNewLog();
ImmutableMemTable = MemTable;
MemTable = new MemTable(this);
force = false;
Compactor.MaybeScheduleCompaction(lockScope);
}
}
}
}
/// <summary>
/// Build a Table file from the contents of *iter. The generated file
/// will be named according to meta->number. On success, the rest of
/// *meta will be filled with metadata about the generated table.
/// If no data is present in *iter, meta->file_size will be set to
/// zero, and no Table file will be produced.
/// </summary>
/// <param name="memTable"></param>
/// <param name="fileNumber"></param>
/// <returns></returns>
public FileMetadata BuildTable(MemTable memTable, ulong fileNumber)
{
CancellationToken.ThrowIfCancellationRequested();
TableBuilder builder = null;
var meta = new FileMetadata
{
FileNumber = fileNumber
};
var tableFileName = FileSystem.GetTableFileName(fileNumber);
try
{
var iterator = memTable.NewIterator();
iterator.SeekToFirst();
if (Log.IsDebugEnabled)
Log.Debug("Writing table with {0:#,#;;00} items to {1}", memTable.Count, tableFileName);
if (iterator.IsValid)
{
var tableFile = FileSystem.NewWritable(tableFileName);
builder = new TableBuilder(this, tableFile, new TemporaryFiles(FileSystem, fileNumber));
meta.SmallestKey = new InternalKey(iterator.Key);
while (iterator.IsValid)
{
CancellationToken.ThrowIfCancellationRequested();
var key = iterator.Key;
meta.LargestKey = new InternalKey(key);
if (Log.IsDebugEnabled)
Log.Debug("Writing item with key {0}", meta.LargestKey.DebugVal);
using (var stream = iterator.CreateValueStream())
builder.Add(key, stream);
iterator.Next();
}
builder.Finish();
meta.FileSize = builder.FileSize;
Debug.Assert(meta.FileSize > 0);
}
}
finally
{
if (builder != null)
builder.Dispose();
if (meta.FileSize == 0)
{
FileSystem.DeleteFile(tableFileName);
}
}
return meta;
}
private ulong RecoverLogFile(ulong logNumber, VersionEdit edit, ulong maxSequence)
{
var logFileName = FileSystem.GetLogFileName(logNumber);
Log.Info("Starting to recover from log: {0}", logFileName);
MemTable mem = null;
using (var logFile = FileSystem.OpenForReading(logFileName))
{
foreach (var item in WriteBatch.ReadFromLog(logFile, Options.BufferPool))
{
var lastSequence = item.WriteSequence + (ulong)item.WriteBatch.OperationCount - 1;
if (lastSequence > maxSequence)
{
maxSequence = lastSequence;
}
if (mem == null)
{
mem = new MemTable(this);
}
item.WriteBatch.Prepare(mem);
item.WriteBatch.Apply(mem, new Reference<ulong> { Value = item.WriteSequence });
if (mem.ApproximateMemoryUsage > Options.WriteBatchSize)
{
Compactor.WriteLevel0Table(mem, null, edit);
mem = null;
}
}
}
if (mem != null)
{
Compactor.WriteLevel0Table(mem, null, edit);
}
return maxSequence;
}
