public void Load(LogReader reader)
{
if (CurrentVersion != null)
{
return;
}
CurrentVersion = ReadVersionEdit(reader);
Log.Debug($"Loading manifest");
Print(Log, CurrentVersion);
foreach (var level in CurrentVersion.Levels) // Search all levels for file with matching index
{
foreach (FileMetadata tbl in level.Value)
{
Table tableReader = tbl.Table;
if (tableReader == null)
{
tableReader = GetTable(tbl.FileNumber);
tableReader.Initialize();
tbl.Table = tableReader;
}
}
}
if (!"leveldb.BytewiseComparator".Equals(CurrentVersion.Comparator, StringComparison.InvariantCultureIgnoreCase))
{
throw new Exception($"Found record, but contains invalid or unsupported comparator: {CurrentVersion.Comparator}");
}
}
public void Load(LogReader reader)
{
if (CurrentVersion != null)
{
return;
}
CurrentVersion = ReadVersionEdit(reader);
Print(CurrentVersion);
foreach (var level in CurrentVersion.NewFiles) // Search all levels for file with matching index
{
foreach (FileMetadata tbl in level.Value)
{
if (Log.IsDebugEnabled)
{
Log.Debug($"Found table file for key in level {level.Key} in file={tbl.FileNumber}");
}
Table tableReader = tbl.Table;
if (tableReader == null)
{
FileInfo f = new FileInfo(Path.Combine(_baseDirectory.FullName, $"{tbl.FileNumber:000000}.ldb"));
if (!f.Exists)
{
throw new Exception($"Could not find table {f.FullName}");
}
tableReader = new Table(f);
tbl.Table = tableReader;
}
}
}
}
internal void Load(LogReader reader)
{
_resultCache = new Dictionary <byte[], ResultCacheEntry>(new ByteArrayComparer());
int entriesCount = 0;
while (true)
{
ReadOnlySpan <byte> data = reader.ReadData();
if (reader.Eof)
{
if (Log.IsDebugEnabled)
{
Log.Debug($"Reached end of stream. No more records to read.");
}
break;
}
var entries = DecodeBatch(data);
entriesCount += entries.Count;
foreach (KeyValuePair <byte[], ResultCacheEntry> entry in entries.OrderBy(kvp => kvp.Value.Sequence))
{
// This should overwrite older entries and only the latest operation should be saved
_resultCache[entry.Key] = entry.Value;
_estimatedSize += (ulong)(entry.Key.Length + 8 + entry.Key.Length + 10 /* varlong * 2 */);
}
}
Log.Debug($"Total count of entries read: {entriesCount}");
Log.Debug($"Total count after filtering entries: {_resultCache.Count}");
}
internal void Load(LogReader reader)
{
//_resultCache = new Dictionary<byte[], ResultCacheEntry>();
_resultCache = new Dictionary <byte[], ResultCacheEntry>(new ByteArrayComparer());
while (true)
{
Record record = reader.ReadRecord();
if (record.LogRecordType == LogRecordType.Eof)
{
if (Log.IsDebugEnabled)
{
Log.Debug($"Reached end of records: {record.ToString()}");
}
break;
}
if (record.LogRecordType != LogRecordType.Full)
{
throw new Exception($"Invalid log file. Didn't find any records. Got record of type {record.LogRecordType}");
}
if (record.Length != (ulong)record.Data.Length)
{
throw new Exception($"Invalid record state. Length not matching");
}
var entries = DecodeBatch(record.Data);
foreach (var entry in entries)
{
//_resultCache.TryAdd(entry.Key, entry.Value);
_resultCache[entry.Key] = entry.Value;
}
}
_resultCache = _resultCache.OrderByDescending(kvp => kvp.Value.Sequence).ToDictionary(k => k.Key, k => k.Value);
}
public static Version ReadVersionEdit(LogReader logReader)
{
var version = new Version();
while (true)
{
ReadOnlySpan <byte> data = logReader.ReadData();
if (logReader.Eof)
{
break;
}
var reader = new SpanReader(data);
//var versionEdit = new VersionEdit();
while (!reader.Eof)
{
var logTag = (LogTagType)reader.ReadVarLong();
switch (logTag)
{
case LogTagType.Comparator:
{
version.Comparator = reader.ReadLengthPrefixedString();
break;
}
case LogTagType.LogNumber:
{
version.LogNumber = reader.ReadVarLong();
break;
}
case LogTagType.NextFileNumber:
{
version.NextFileNumber = reader.ReadVarLong();
break;
}
case LogTagType.LastSequence:
{
version.LastSequenceNumber = reader.ReadVarLong();
break;
}
case LogTagType.CompactPointer:
{
int level = (int)reader.ReadVarLong();
var internalKey = reader.ReadLengthPrefixedBytes();
version.CompactPointers[level] = internalKey.ToArray();
break;
}
case LogTagType.DeletedFile:
{
int level = (int)reader.ReadVarLong();
ulong fileNumber = reader.ReadVarLong();
if (!version.DeletedFiles.ContainsKey(level))
{
version.DeletedFiles[level] = new List <ulong>();
}
version.DeletedFiles[level].Add(fileNumber);
break;
}
case LogTagType.NewFile:
{
int level = (int)reader.ReadVarLong();
ulong fileNumber = reader.ReadVarLong();
ulong fileSize = reader.ReadVarLong();
ReadOnlySpan <byte> smallest = reader.ReadLengthPrefixedBytes();
ReadOnlySpan <byte> largest = reader.ReadLengthPrefixedBytes();
var fileMetadata = new FileMetadata();
fileMetadata.FileNumber = fileNumber;
fileMetadata.FileSize = fileSize;
fileMetadata.SmallestKey = smallest.ToArray();
fileMetadata.LargestKey = largest.ToArray();
if (!version.Levels.ContainsKey(level))
{
version.Levels[level] = new List <FileMetadata>();
}
version.Levels[level].Add(fileMetadata);
break;
}
case LogTagType.PrevLogNumber:
{
version.PreviousLogNumber = reader.ReadVarLong();
break;
}
default:
{
throw new ArgumentOutOfRangeException($"Unknown tag={logTag}");
}
}
}
}
// Clean files
var deletedFiles = new List <ulong>();
foreach (List <ulong> versionDeletedFile in version.DeletedFiles.Values)
{
deletedFiles.AddRange(versionDeletedFile);
}
foreach (KeyValuePair <int, List <FileMetadata> > levelKvp in version.Levels)
{
foreach (FileMetadata newFile in levelKvp.Value.ToArray())
{
if (deletedFiles.Contains(newFile.FileNumber))
{
levelKvp.Value.Remove(newFile);
}
}
}
version.Levels = version.Levels.OrderBy(kvp => kvp.Key).ToDictionary(pair => pair.Key, pair => pair.Value);
version.Comparator ??= "leveldb.BytewiseComparator";
return(version);
}
public VersionEdit ReadVersionEdit(LogReader logReader)
{
string comparator = null;
ulong? logNumber = null;
ulong? previousLogNumber = null;
ulong? nextFileNumber = null;
ulong? lastSequenceNumber = null;
VersionEdit finalVersion = new VersionEdit();
while (true)
{
Record record = logReader.ReadRecord();
if (record.LogRecordType != LogRecordType.Full)
{
break;
}
var reader = new SpanReader(record.Data);
VersionEdit versionEdit = new VersionEdit();
while (!reader.Eof)
{
LogTagType logTag = (LogTagType)reader.ReadVarLong();
switch (logTag)
{
case LogTagType.Comparator:
{
versionEdit.Comparator = reader.ReadLengthPrefixedString();
break;
}
case LogTagType.LogNumber:
{
versionEdit.LogNumber = reader.ReadVarLong();
break;
}
case LogTagType.NextFileNumber:
{
versionEdit.NextFileNumber = reader.ReadVarLong();
break;
}
case LogTagType.LastSequence:
{
versionEdit.LastSequenceNumber = reader.ReadVarLong();
break;
}
case LogTagType.CompactPointer:
{
int level = (int)reader.ReadVarLong();
var internalKey = reader.ReadLengthPrefixedBytes();
versionEdit.CompactPointers[level] = internalKey.ToArray();
break;
}
case LogTagType.DeletedFile:
{
int level = (int)reader.ReadVarLong();
ulong fileNumber = reader.ReadVarLong();
if (!versionEdit.DeletedFiles.ContainsKey(level))
{
versionEdit.DeletedFiles[level] = new List <ulong>();
}
versionEdit.DeletedFiles[level].Add(fileNumber);
if (!finalVersion.DeletedFiles.ContainsKey(level))
{
finalVersion.DeletedFiles[level] = new List <ulong>();
}
finalVersion.DeletedFiles[level].Add(fileNumber);
break;
}
case LogTagType.NewFile:
{
int level = (int)reader.ReadVarLong();
ulong fileNumber = reader.ReadVarLong();
ulong fileSize = reader.ReadVarLong();
var smallest = reader.ReadLengthPrefixedBytes();
var largest = reader.ReadLengthPrefixedBytes();
FileMetadata fileMetadata = new FileMetadata();
fileMetadata.FileNumber = fileNumber;
fileMetadata.FileSize = fileSize;
fileMetadata.SmallestKey = smallest.ToArray();
fileMetadata.LargestKey = largest.ToArray();
if (!versionEdit.NewFiles.ContainsKey(level))
{
versionEdit.NewFiles[level] = new List <FileMetadata>();
}
versionEdit.NewFiles[level].Add(fileMetadata);
if (!finalVersion.NewFiles.ContainsKey(level))
{
finalVersion.NewFiles[level] = new List <FileMetadata>();
}
finalVersion.NewFiles[level].Add(fileMetadata);
break;
}
case LogTagType.PrevLogNumber:
{
versionEdit.PreviousLogNumber = reader.ReadVarLong();
break;
}
default:
{
throw new ArgumentOutOfRangeException($"Unknown tag={logTag}");
}
}
}
versionEdit.CompactPointers = versionEdit.CompactPointers.Count == 0 ? null : versionEdit.CompactPointers;
versionEdit.DeletedFiles = versionEdit.DeletedFiles.Count == 0 ? null : versionEdit.DeletedFiles;
versionEdit.NewFiles = versionEdit.NewFiles.Count == 0 ? null : versionEdit.NewFiles;
comparator = versionEdit.Comparator ?? comparator;
logNumber = versionEdit.LogNumber ?? logNumber;
previousLogNumber = versionEdit.PreviousLogNumber ?? previousLogNumber;
nextFileNumber = versionEdit.NextFileNumber ?? nextFileNumber;
lastSequenceNumber = versionEdit.LastSequenceNumber ?? lastSequenceNumber;
}
// Clean files
List <ulong> deletedFiles = new List <ulong>();
foreach (var versionDeletedFile in finalVersion.DeletedFiles.Values)
{
deletedFiles.AddRange(versionDeletedFile);
}
foreach (var levelKvp in finalVersion.NewFiles)
{
foreach (var newFile in levelKvp.Value.ToArray())
{
if (deletedFiles.Contains(newFile.FileNumber))
{
levelKvp.Value.Remove(newFile);
}
}
}
finalVersion.NewFiles = finalVersion.NewFiles.OrderBy(kvp => kvp.Key).ToDictionary(pair => pair.Key, pair => pair.Value);
finalVersion.Comparator = comparator;
finalVersion.LogNumber = logNumber;
finalVersion.PreviousLogNumber = previousLogNumber;
finalVersion.NextFileNumber = nextFileNumber;
finalVersion.LastSequenceNumber = lastSequenceNumber;
return(finalVersion);
}
public void Open()
{
if (_dbLock == null)
{
throw new ObjectDisposedException("Database was closed and can not be reopened");
}
_dbLock.EnterWriteLock();
try
{
if (_manifest != null)
{
throw new InvalidOperationException("Already had manifest for database. Did you already open it?");
}
if (_memCache != null)
{
throw new InvalidOperationException("Already had memory cache for database. Did you already open it?");
}
if (Directory.Name.EndsWith(".mcworld"))
{
// Exported from MCPE. Unpack to temp
Log.Debug($"Opening directory: {Directory.Name}");
var originalFile = Directory;
string newDirPath = Path.Combine(Path.GetTempPath(), Directory.Name);
Directory = new DirectoryInfo(Path.Combine(newDirPath, "db"));
if (!Directory.Exists || originalFile.LastWriteTimeUtc > Directory.LastWriteTimeUtc)
{
ZipFile.ExtractToDirectory(originalFile.FullName, newDirPath, true);
Log.Warn($"Created new temp directory: {Directory.FullName}");
}
Directory.Refresh();
Log.Warn($"Extracted bedrock world and set new DB directory to: {Directory.FullName}");
}
// Verify that directory exists
if (!Directory.Exists)
{
Directory.Create();
Directory.Refresh();
}
if (!File.Exists(GetCurrentFileName()))
{
if (!_createIfMissing)
{
var notFoundException = new DirectoryNotFoundException(Directory.FullName);
Log.Error(notFoundException);
throw notFoundException;
}
// Create new MANIFEST
var manifest = new Manifest(Directory);
manifest.CurrentVersion = new Version()
{
Comparator = "leveldb.BytewiseComparator",
LogNumber = 1,
PreviousLogNumber = 0,
NextFileNumber = 2,
LastSequenceNumber = 0
};
var manifestFileInfo = new FileInfo(GetManifestFileName(1));
if (manifestFileInfo.Exists)
{
throw new PanicException($"Trying to create database, but found existing MANIFEST file at {manifestFileInfo.FullName}. Aborting.");
}
using var writer = new LogWriter(manifestFileInfo);
manifest.Save(writer);
manifest.Close();
// Create new CURRENT text file and store manifest filename in it
using StreamWriter current = File.CreateText(GetCurrentFileName());
current.WriteLine(manifestFileInfo.Name);
current.Close();
// Done and created
}
Directory.Refresh(); // If this has been manipulated on the way, this is really needed.
// Read Manifest into memory
string manifestFilename = GetManifestFileNameFromCurrent();
Log.Debug($"Reading manifest from {manifestFilename}");
using (var reader = new LogReader(new FileInfo(manifestFilename)))
{
_manifest = new Manifest(Directory);
_manifest.Load(reader);
}
// Read current log
var logFile = new FileInfo(GetLogFileName(_manifest.CurrentVersion.LogNumber));
Log.Debug($"Reading log from {logFile.FullName}");
using (var reader = new LogReader(logFile))
{
_memCache = new MemCache();
_memCache.Load(reader);
}
// Append mode
_log = new LogWriter(logFile);
}
finally
{
_dbLock.ExitWriteLock();
}
// We do this on startup. It will rotate the log files and create
// level 0 tables. However, we want to use into reusing the logs.
CompactMemCache();
CleanOldFiles();
_compactTimer = new Timer(state => DoCompaction(), null, 300, 300);
}
public void Open()
{
if (_manifest != null)
{
throw new InvalidOperationException("Already had manifest for database. Did you already open it?");
}
if (_newMemCache != null)
{
throw new InvalidOperationException("Already had memory cache for database. Did you already open it?");
}
if (Directory.Name.EndsWith(".mcworld"))
{
// Exported from MCPE. Unpack to temp
Log.Debug($"Opening directory: {Directory.Name}");
var originalFile = Directory;
string newDirPath = Path.Combine(Path.GetTempPath(), Directory.Name);
Directory = new DirectoryInfo(Path.Combine(newDirPath, "db"));
if (!Directory.Exists || originalFile.LastWriteTimeUtc > Directory.LastWriteTimeUtc)
{
ZipFile.ExtractToDirectory(originalFile.FullName, newDirPath, true);
Log.Warn($"Created new temp directory: {Directory.FullName}");
}
Log.Warn($"Extracted bedrock world and set new DB directory to: {Directory.FullName}");
}
// Verify that directory exists
if (!Directory.Exists)
{
if (!CreateIfMissing)
{
throw new DirectoryNotFoundException(Directory.Name);
}
Directory.Create();
// Create new MANIFEST
VersionEdit newVersion = new VersionEdit();
newVersion.NextFileNumber = 1;
newVersion.Comparator = "leveldb.BytewiseComparator";
// Create new CURRENT text file and store manifest filename in it
using (var manifestStream = File.CreateText($@"{Path.Combine(Directory.FullName, "CURRENT")}"))
{
manifestStream.WriteLine($"MANIFEST-{newVersion.NextFileNumber++:000000}");
manifestStream.Close();
}
// Done
}
// Read Manifest into memory
string manifestFilename;
using (var currentStream = File.OpenText($@"{Path.Combine(Directory.FullName, "CURRENT")}"))
{
manifestFilename = currentStream.ReadLine();
currentStream.Close();
}
Log.Debug($"Reading manifest from {Path.Combine(Directory.FullName, manifestFilename)}");
using (var reader = new LogReader(new FileInfo($@"{Path.Combine(Directory.FullName, manifestFilename)}")))
{
_manifest = new Manifest(Directory);
_manifest.Load(reader);
}
// Read current log
//TODO: remove unit-test-stuff
var logFileName = Path.Combine(Directory.FullName, $"{_manifest.CurrentVersion.LogNumber + 1:000000}.log");
FileInfo f = new FileInfo(logFileName);
if (!f.Exists)
{
f = new FileInfo(Path.Combine(Directory.FullName, $"{_manifest.CurrentVersion.LogNumber:000000}.log"));
}
using (var reader = new LogReader(f))
{
_newMemCache = new MemCache();
_newMemCache.Load(reader);
}
}
