internal FileMetadata WriteLevel0Table(MemCache memCache, FileInfo tableFileInfo)
{
using FileStream fileStream = File.Create(tableFileInfo.FullName);
var creator = new TableCreator(fileStream);
byte[] smallestKey = null;
byte[] largestKey = null;
foreach (KeyValuePair <byte[], MemCache.ResultCacheEntry> entry in memCache._resultCache.OrderBy(kvp => kvp.Key, new BytewiseComparator()).ThenBy(kvp => kvp.Value.Sequence))
{
if (entry.Value.ResultState != ResultState.Exist && entry.Value.ResultState != ResultState.Deleted)
{
continue;
}
byte[] opAndSeq = BitConverter.GetBytes((ulong)entry.Value.Sequence << 8);
opAndSeq[0] = (byte)(entry.Value.ResultState == ResultState.Exist ? 1 : 0);
byte[] key = entry.Key.Concat(opAndSeq).ToArray();
byte[] data = entry.Value.Data;
smallestKey ??= key;
largestKey = key;
//if (Log.IsDebugEnabled)
//{
// if (entry.Value.ResultState == ResultState.Deleted)
// Log.Warn($"Key:{key.ToHexString()} {entry.Value.Sequence}, {entry.Value.ResultState == ResultState.Exist}, size:{entry.Value.Data?.Length ?? 0}");
// else
//Log.Debug($"Key: {entry.Key.ToHexString()} FullKey:{key.ToHexString()} Seq: {entry.Value.Sequence}, Exist: {entry.Value.ResultState == ResultState.Exist}");
//}
creator.Add(key, data);
}
creator.Finish();
fileStream.Flush();
fileStream.Close();
tableFileInfo.Refresh();
Log.Debug($"Size distinct:{memCache._resultCache.Distinct().Count()}");
Log.Debug($"Wrote {memCache._resultCache.Count} values to {tableFileInfo.Name}");
return(new FileMetadata
{
FileNumber = 0, // Set in calling method
FileSize = (ulong)tableFileInfo.Length,
SmallestKey = smallestKey,
LargestKey = largestKey,
Table = new Table(tableFileInfo)
});
}
public byte[] Get(Span <byte> key)
{
_dbLock.EnterReadLock();
try
{
if (_manifest == null)
{
throw new InvalidOperationException("No manifest for database. Did you open it?");
}
if (_memCache == null)
{
throw new InvalidOperationException("No current memory cache for database. Did you open it?");
}
ResultStatus result = _memCache.Get(key);
if (result.State == ResultState.Deleted || result.State == ResultState.Exist)
{
if (result.Data == ReadOnlySpan <byte> .Empty)
{
return(null);
}
return(result.Data.ToArray());
}
MemCache imm = _immutableMemCache;
if (imm != null)
{
result = imm.Get(key);
if (result.State == ResultState.Deleted || result.State == ResultState.Exist)
{
if (result.Data == ReadOnlySpan <byte> .Empty)
{
return(null);
}
return(result.Data.ToArray());
}
}
result = _manifest.Get(key);
if (result.Data == ReadOnlySpan <byte> .Empty)
{
return(null);
}
return(result.Data.ToArray());
}
finally
{
_dbLock.ExitReadLock();
}
}
private void MakeSurePutWorks()
{
if (Options.ReadOnly)
{
return;
}
if (!_dbLock.IsWriteLockHeld)
{
throw new SynchronizationLockException("Expected caller to hold write lock");
}
if (_memCache.GetEstimatedSize() < Options.MaxMemCacheSize)
{
return; // All fine, carry on
}
if (_immutableMemCache != null)
{
// still compacting
//_compactReset.WaitOne();
}
else
{
// Rotate memcache and log
Log.Debug($"Time to rotate memcache. Size={_memCache.GetEstimatedSize()} bytes");
LogWriter oldLog = _log;
ulong logNumber = _manifest.CurrentVersion.GetNewFileNumber();
_log = new LogWriter(new FileInfo(GetLogFileName(logNumber)));
_logNumber = logNumber;
_immutableMemCache = _memCache;
_memCache = new MemCache();
oldLog.Close();
_compactTask = new Task(CompactMemCache);
_compactTask.Start();
// Schedule compact
}
}
public void Close()
{
if (_dbLock == null)
{
return;
}
_compactTask?.Wait();
_dbLock?.EnterWriteLock();
if (_dbLock == null)
{
return;
}
try
{
if (_compactTimer != null)
{
var timeFinishWaiter = new ManualResetEvent(false);
_compactTimer.Dispose(timeFinishWaiter);
timeFinishWaiter.WaitOne();
}
_memCache = null;
_log?.Close();
_log = null;
_manifest?.Close();
_manifest = null;
}
finally
{
var dbLock = _dbLock;
_dbLock = null;
dbLock.ExitWriteLock();
dbLock?.Dispose();
}
}
public void Close()
{
if (_newMemCache != null)
{
var memCache = _newMemCache;
_newMemCache = null;
if (_manifest != null)
{
var nextLogNumber = _manifest.CurrentVersion.LogNumber + 1;
using (var logWriter = new LogWriter(new FileInfo(Path.Combine(Directory.FullName, $"{nextLogNumber:000000}.log"))))
{
memCache.Write(logWriter);
}
}
}
if (_manifest != null)
{
var temp = _manifest;
_manifest = null;
temp.Close();
}
}
private void CompactMemCache()
{
if (Options.ReadOnly)
{
return;
}
Log.Debug($"Checking if we should compact");
_dbLock.EnterWriteLock();
try
{
//if (!force && _memCache.GetEstimatedSize() < Options.MaxMemCacheSize) return; // 4Mb
if (_immutableMemCache == null)
{
return;
}
Log.Debug($"Compact kicking in");
_compactReset.Reset();
// Write immutable memcache to a level 0 table
Version version = _manifest.CurrentVersion;
ulong newFileNumber = version.GetNewFileNumber();
var tableFileInfo = new FileInfo(GetTableFileName(newFileNumber));
FileMetadata meta = WriteLevel0Table(_immutableMemCache, tableFileInfo);
meta.FileNumber = newFileNumber;
Level0Tables.Add(meta);
// Update version data and commit new manifest (save)
var newVersion = new Version(version);
newVersion.LogNumber = _logNumber;
newVersion.AddFile(0, meta);
Manifest.Print(Log, newVersion);
_immutableMemCache = null;
// Update manifest with new version
_manifest.CurrentVersion = newVersion;
var manifestFileName = new FileInfo(GetManifestFileName(newVersion.GetNewFileNumber()));
using (var writer = new LogWriter(manifestFileName))
{
_manifest.Save(writer);
}
using (StreamWriter currentStream = File.CreateText(GetCurrentFileName()))
{
currentStream.WriteLine(manifestFileName.Name);
currentStream.Close();
}
CleanOldFiles();
_compactTask = null;
_compactReset.Set();
}
finally
{
_dbLock.ExitWriteLock();
}
}
public void Open()
{
if (_dbLock == null)
{
throw new ObjectDisposedException("Database was closed and can not be reopened");
}
_dbLock.EnterWriteLock();
try
{
if (_manifest != null)
{
throw new InvalidOperationException("Already had manifest for database. Did you already open it?");
}
if (_memCache != null)
{
throw new InvalidOperationException("Already had memory cache for database. Did you already open it?");
}
if (Directory.Name.EndsWith(".mcworld"))
{
// Exported from MCPE. Unpack to temp
Log.Debug($"Opening directory: {Directory.Name}");
var originalFile = Directory;
string newDirPath = Path.Combine(Path.GetTempPath(), Directory.Name);
Directory = new DirectoryInfo(Path.Combine(newDirPath, "db"));
if (!Directory.Exists || originalFile.LastWriteTimeUtc > Directory.LastWriteTimeUtc)
{
ZipFile.ExtractToDirectory(originalFile.FullName, newDirPath, true);
Log.Warn($"Created new temp directory: {Directory.FullName}");
}
Directory.Refresh();
Log.Warn($"Extracted bedrock world and set new DB directory to: {Directory.FullName}");
}
// Verify that directory exists
if (!Directory.Exists)
{
Directory.Create();
Directory.Refresh();
}
if (!File.Exists(GetCurrentFileName()))
{
if (!_createIfMissing)
{
var notFoundException = new DirectoryNotFoundException(Directory.FullName);
Log.Error(notFoundException);
throw notFoundException;
}
// Create new MANIFEST
var manifest = new Manifest(Directory);
manifest.CurrentVersion = new Version()
{
Comparator = "leveldb.BytewiseComparator",
LogNumber = 1,
PreviousLogNumber = 0,
NextFileNumber = 2,
LastSequenceNumber = 0
};
var manifestFileInfo = new FileInfo(GetManifestFileName(1));
if (manifestFileInfo.Exists)
{
throw new PanicException($"Trying to create database, but found existing MANIFEST file at {manifestFileInfo.FullName}. Aborting.");
}
using var writer = new LogWriter(manifestFileInfo);
manifest.Save(writer);
manifest.Close();
// Create new CURRENT text file and store manifest filename in it
using StreamWriter current = File.CreateText(GetCurrentFileName());
current.WriteLine(manifestFileInfo.Name);
current.Close();
// Done and created
}
Directory.Refresh(); // If this has been manipulated on the way, this is really needed.
// Read Manifest into memory
string manifestFilename = GetManifestFileNameFromCurrent();
Log.Debug($"Reading manifest from {manifestFilename}");
using (var reader = new LogReader(new FileInfo(manifestFilename)))
{
_manifest = new Manifest(Directory);
_manifest.Load(reader);
}
// Read current log
var logFile = new FileInfo(GetLogFileName(_manifest.CurrentVersion.LogNumber));
Log.Debug($"Reading log from {logFile.FullName}");
using (var reader = new LogReader(logFile))
{
_memCache = new MemCache();
_memCache.Load(reader);
}
// Append mode
_log = new LogWriter(logFile);
}
finally
{
_dbLock.ExitWriteLock();
}
// We do this on startup. It will rotate the log files and create
// level 0 tables. However, we want to use into reusing the logs.
CompactMemCache();
CleanOldFiles();
_compactTimer = new Timer(state => DoCompaction(), null, 300, 300);
}
public void Open()
{
if (_manifest != null)
{
throw new InvalidOperationException("Already had manifest for database. Did you already open it?");
}
if (_newMemCache != null)
{
throw new InvalidOperationException("Already had memory cache for database. Did you already open it?");
}
if (Directory.Name.EndsWith(".mcworld"))
{
// Exported from MCPE. Unpack to temp
Log.Debug($"Opening directory: {Directory.Name}");
var originalFile = Directory;
string newDirPath = Path.Combine(Path.GetTempPath(), Directory.Name);
Directory = new DirectoryInfo(Path.Combine(newDirPath, "db"));
if (!Directory.Exists || originalFile.LastWriteTimeUtc > Directory.LastWriteTimeUtc)
{
ZipFile.ExtractToDirectory(originalFile.FullName, newDirPath, true);
Log.Warn($"Created new temp directory: {Directory.FullName}");
}
Log.Warn($"Extracted bedrock world and set new DB directory to: {Directory.FullName}");
}
// Verify that directory exists
if (!Directory.Exists)
{
if (!CreateIfMissing)
{
throw new DirectoryNotFoundException(Directory.Name);
}
Directory.Create();
// Create new MANIFEST
VersionEdit newVersion = new VersionEdit();
newVersion.NextFileNumber = 1;
newVersion.Comparator = "leveldb.BytewiseComparator";
// Create new CURRENT text file and store manifest filename in it
using (var manifestStream = File.CreateText($@"{Path.Combine(Directory.FullName, "CURRENT")}"))
{
manifestStream.WriteLine($"MANIFEST-{newVersion.NextFileNumber++:000000}");
manifestStream.Close();
}
// Done
}
// Read Manifest into memory
string manifestFilename;
using (var currentStream = File.OpenText($@"{Path.Combine(Directory.FullName, "CURRENT")}"))
{
manifestFilename = currentStream.ReadLine();
currentStream.Close();
}
Log.Debug($"Reading manifest from {Path.Combine(Directory.FullName, manifestFilename)}");
using (var reader = new LogReader(new FileInfo($@"{Path.Combine(Directory.FullName, manifestFilename)}")))
{
_manifest = new Manifest(Directory);
_manifest.Load(reader);
}
// Read current log
//TODO: remove unit-test-stuff
var logFileName = Path.Combine(Directory.FullName, $"{_manifest.CurrentVersion.LogNumber + 1:000000}.log");
FileInfo f = new FileInfo(logFileName);
if (!f.Exists)
{
f = new FileInfo(Path.Combine(Directory.FullName, $"{_manifest.CurrentVersion.LogNumber:000000}.log"));
}
using (var reader = new LogReader(f))
{
_newMemCache = new MemCache();
_newMemCache.Load(reader);
}
}