internal SSTable(Stream stream, IBlockCache?cache, PlaneDBOptions options)
{
this.stream = stream;
comparer = options.Comparer;
reader = OpenReaderStream(cache, options);
index = new Lazy <Index>(() => new Index(reader), LazyThreadSafetyMode.ExecutionAndPublication);
}
internal SSTableBuilder(Stream stream, PlaneDBOptions options)
{
writer = new BlockWriteOnceStream(stream, options.BlockTransformer);
comparer = options.Comparer;
fullySync = options.MaxJournalActions < 0;
dictionary = new SortedList <byte[], byte[]?>(comparer);
}
/// <param name="location">Directory that will store the PlaneDB</param>
/// <param name="mode">File mode to use, supported are: CreateNew, Open (existing), OpenOrCreate</param>
/// <param name="options">Options to use, such as the transformer, cache settings, etc.</param>
/// <summary>Opens or creates a new PlaneDB.</summary>
public PlaneDB(DirectoryInfo location, FileMode mode, PlaneDBOptions options)
{
options.Validate();
Location = location;
this.options = options.Clone();
blockCache = new BlockCache(options.BlockCacheCapacity);
memoryTable = new MemoryTable(options);
if (mode == FileMode.CreateNew || mode == FileMode.OpenOrCreate)
{
location.Create();
}
state = new PlaneDBState(location, mode, options);
ReopenSSTables();
if (tables.Count(i => i.Value.DiskSize < 524288) > 2)
{
MaybeMerge(true);
}
if (!options.ThreadSafe)
{
return;
}
mergeThread = new Thread(MergeLoop)
{
Priority = ThreadPriority.BelowNormal, Name = "Plane-Background-Merge"
};
mergeThread.Start();
}
/// <summary>
/// Create a new typed Key-Value store
/// </summary>
/// <remarks>
/// Please note that the internal sort order will still be based upon the byte-array comparer
/// </remarks>
/// <param name="keySerializer">Serializer to use to handle keys</param>
/// <param name="valueSerializer">Serializer to use to handle values</param>
/// <param name="location">Directory that will store the PlaneDB</param>
/// <param name="mode">File mode to use, supported are: CreateNew, Open (existing), OpenOrCreate</param>
/// <param name="options">Options to use, such as the transformer, cache settings, etc.</param>
public TypedPlaneDB(ISerializer <TKey> keySerializer, ISerializer <TValue> valueSerializer, DirectoryInfo location,
FileMode mode, PlaneDBOptions options)
{
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
wrapped = new PlaneDB(location, mode, options);
wrapped.OnFlushMemoryTable += (sender, db) => OnFlushMemoryTable?.Invoke(this, this);
wrapped.OnMergedTables += (sender, db) => OnMergedTables?.Invoke(this, this);
}
internal Journal(Stream stream, PlaneDBOptions options)
{
this.stream = stream;
transformer = options.BlockTransformer;
fullySync = options.MaxJournalActions < 0;
maxActions = Math.Max(0, options.MaxJournalActions);
stream.WriteInt32(Constants.MAGIC);
flusher = Task.Factory.StartNew(RunFlushLoop, cancel.Token, TaskCreationOptions.LongRunning,
TaskScheduler.Current);
}
internal PlaneDBState(DirectoryInfo location, FileMode mode, PlaneDBOptions options)
{
this.location = location;
this.options = options;
ReadWriteLock = options.ThreadSafe ? options.TrueReadWriteLock : new FakeReadWriteLock();
try {
lockFile = mode switch {
FileMode.CreateNew => new FileStream(Manifest.FindFile(location, options, Manifest.LOCK_FILE).FullName,
FileMode.CreateNew, FileAccess.ReadWrite,
FileShare.None),
FileMode.Open => new FileStream(Manifest.FindFile(location, options, Manifest.LOCK_FILE).FullName,
FileMode.Create, FileAccess.ReadWrite,
FileShare.None),
FileMode.OpenOrCreate => new FileStream(Manifest.FindFile(location, options, Manifest.LOCK_FILE).FullName,
FileMode.Create, FileAccess.ReadWrite,
FileShare.None),
_ => throw new ArgumentOutOfRangeException(nameof(mode), mode, null)
};
}
catch (UnauthorizedAccessException ex) {
throw new AlreadyLockedException(ex);
}
catch (IOException ex) {
throw new AlreadyLockedException(ex);
}
// ReSharper disable once ConvertSwitchStatementToSwitchExpression
switch (mode)
{
case FileMode.CreateNew:
case FileMode.Open:
case FileMode.OpenOrCreate:
Manifest = new Manifest(location, mode, options);
break;
case FileMode.Append:
case FileMode.Create:
case FileMode.Truncate:
throw new NotSupportedException(nameof(mode));
default:
throw new ArgumentOutOfRangeException(nameof(mode), mode, null);
}
Manifest.RemoveOrphans();
MaybeReplayJournal(Manifest);
Journal = OpenJournal();
}
/// <summary>
/// Create a new String/String Key-Value store
/// </summary>
/// <param name="location">Directory that will store the PlaneDB</param>
/// <param name="mode">File mode to use, supported are: CreateNew, Open (existing), OpenOrCreate</param>
/// <param name="options">Options to use, such as the transformer, cache settings, etc.</param>
public StringPlaneDB(DirectoryInfo location, FileMode mode, PlaneDBOptions options)
: base(new StringSerializer(), new StringSerializer(), location, mode, options)
{
}
internal static void ReplayOnto(Stream journal, PlaneDBOptions options, IWriteOnlyTable table)
{
var transformer = options.BlockTransformer;
var actions = 0;
journal.Seek(0, SeekOrigin.Begin);
if (journal.ReadInt32() != Constants.MAGIC)
{
throw new IOException("Bad journal file (wrong blocktransformer?)");
}
Span <byte> small = stackalloc byte[4096];
for (;;)
{
try {
var unlength = journal.ReadInt32();
var length = journal.ReadInt32();
if (length <= 0 || unlength <= 0)
{
throw new IOException("Bad record");
}
Span <byte> input = length <= 4096 ? small.Slice(0, length) : new byte[length];
Span <byte> buffer = unlength <= 4096 ? small.Slice(0, unlength) : new byte[unlength];
journal.ReadFullBlock(input);
var tlen = transformer.UntransformBlock(input, buffer);
if (tlen <= 0)
{
throw new IOException($"Bad record ({length}/{tlen})");
}
buffer = buffer.Slice(0, tlen);
var type = buffer[0];
buffer = buffer.Slice(1);
switch ((RecordType)type)
{
case RecordType.Put: {
var klen = BinaryPrimitives.ReadInt32LittleEndian(buffer);
var vlen = BinaryPrimitives.ReadInt32LittleEndian(buffer.Slice(sizeof(int)));
var key = buffer.Slice(sizeof(int) * 2, klen);
var val = buffer.Slice(sizeof(int) * 2 + klen, vlen);
table.Put(key, val);
++actions;
break;
}
case RecordType.Remove: {
var klen = BinaryPrimitives.ReadInt32LittleEndian(buffer);
var key = buffer.Slice(sizeof(int), klen);
table.Remove(key);
++actions;
break;
}
case RecordType.Update: {
var klen = BinaryPrimitives.ReadInt32LittleEndian(buffer);
var vlen = BinaryPrimitives.ReadInt32LittleEndian(buffer.Slice(sizeof(int)));
var key = buffer.Slice(sizeof(int) * 2, klen);
var val = buffer.Slice(sizeof(int) * 2 + klen, vlen);
table.Update(key, val);
++actions;
break;
}
default:
throw new ArgumentOutOfRangeException();
}
}
catch (IOException) {
break;
}
}
if (actions > 0)
{
return;
}
throw journal switch {
FileStream fs => new BrokenJournalException(fs),
_ => new BrokenJournalException()
};
}
private BlockReadOnlyStream OpenReaderStream(IBlockCache?cache, PlaneDBOptions options)
{
return(new BlockReadOnlyStream(stream, options.BlockTransformer, cache: cache));
}
/// <summary>
/// Creates a new typed persistent set
/// </summary>
/// <param name="serializer">Serializer to use</param>
/// <param name="location">Directory that will store the PlaneSet</param>
/// <param name="mode">File mode to use, supported are: CreateNew, Open (existing), OpenOrCreate</param>
/// <param name="options">Options to use, such as the transformer, cache settings, etc.</param>
public TypedPlaneSet(ISerializer <T> serializer, DirectoryInfo location, FileMode mode, PlaneDBOptions options)
{
this.serializer = serializer;
wrapped = new PlaneSet(location, mode, options);
}
private Manifest(DirectoryInfo location, Stream stream, PlaneDBOptions options, ulong counter)
{
this.location = location;
this.counter = counter;
this.stream = stream;
this.options = options;
if (stream.Length == 0)
{
InitEmpty();
return;
}
stream.Seek(0, SeekOrigin.Begin);
if (stream.ReadInt32() != Constants.MAGIC)
{
throw new IOException("Bad manifest magic");
}
this.counter = stream.ReadUInt64();
var magic2Length = stream.ReadInt32();
if (magic2Length < 0 || magic2Length > Int16.MaxValue)
{
throw new BadMagicException();
}
var magic2 = stream.ReadFullBlock(magic2Length);
Span <byte> actual = stackalloc byte[1024];
int alen;
try {
alen = options.BlockTransformer.UntransformBlock(magic2, actual);
}
catch {
throw new BadMagicException();
}
if (alen != Constants.MagicBytes.Length || !actual.Slice(0, alen).SequenceEqual(Constants.MagicBytes))
{
throw new BadMagicException();
}
for (;;)
{
var level = stream.ReadByte();
if (level < 0)
{
break;
}
var count = stream.ReadInt32();
byte[] name = Array.Empty <byte>();
if (count < 0)
{
if (count == int.MinValue)
{
count = 0;
}
else
{
count = -count;
}
var namelen = stream.ReadInt32();
name = stream.ReadFullBlock(namelen);
}
if (count == 0)
{
GetLevel(name).Remove((byte)level);
continue;
}
var items = Enumerable.Range(0, count).Select(_ => stream.ReadUInt64()).OrderBy(i => i).ToArray();
EnsureLevel(name)[(byte)level] = items;
}
}
internal Manifest(DirectoryInfo location, Stream stream, PlaneDBOptions options)
: this(location, stream, options, 0)
{
}
internal Manifest(DirectoryInfo location, FileMode mode, PlaneDBOptions options)
: this(location, OpenManifestStream(location, options, mode), options, 0)
{
}
private static FileStream OpenManifestStream(DirectoryInfo location, PlaneDBOptions options, FileMode mode)
{
return(new FileStream(FindFile(location, options, MANIFEST_FILE).FullName, mode, FileAccess.ReadWrite,
FileShare.None, 4096));
}
internal static FileInfo FindFile(DirectoryInfo location, PlaneDBOptions options, string filename)
{
var ts = IsNullOrEmpty(options.TableSpace) ? "default" : options.TableSpace;
return(new FileInfo(Path.Combine(location.FullName, $"{ts}-{filename}.planedb")));
}
/// <param name="location">Directory that will store the PlaneSet</param>
/// <param name="mode">File mode to use, supported are: CreateNew, Open (existing), OpenOrCreate</param>
/// <param name="options">Options to use, such as the transformer, cache settings, etc.</param>
/// <summary>Opens or creates a new PlaneSet</summary>
public PlaneSet(DirectoryInfo location, FileMode mode, PlaneDBOptions options)
{
wrappeDB = new PlaneDB(location, mode, options);
}