public void WriteSnapshot(LogWriter logWriter, VersionSet versionSet)
{
var edit = new VersionEdit();
AddMetadata(edit, storageContext.Options);
AddCompactionPointers(edit, versionSet);
AddFiles(edit, versionSet);
edit.EncodeTo(logWriter);
}
private static void AddCompactionPointers(VersionEdit edit, VersionSet versionSet)
{
for (int level = 0; level < Config.NumberOfLevels; level++)
{
var compactionPointer = versionSet.CompactionPointers[level];
if (!compactionPointer.IsEmpty())
{
edit.SetCompactionPointer(level, new InternalKey(compactionPointer));
}
}
}
private static void AddFiles(VersionEdit edit, VersionSet versionSet)
{
for (int level = 0; level < Config.NumberOfLevels; level++)
{
var files = versionSet.Current.Files[level];
foreach (var file in files)
{
edit.AddFile(level, file);
}
}
}
public static VersionEdit DecodeFrom(Stream stream)
{
var result = new VersionEdit();
while (true)
{
Tag tag;
try
{
tag = (Tag)stream.Read7BitEncodedInt();
}
catch (EndOfStreamException)
{
break;
}
int level;
switch (tag)
{
case Tag.Comparator:
var slice = stream.ReadLengthPrefixedSlice();
result.SetComparatorName(slice);
break;
case Tag.LogNumber:
result.SetLogNumber((ulong)stream.Read7BitEncodedLong());
break;
case Tag.PrevLogNumber:
result.SetPrevLogNumber((ulong)stream.Read7BitEncodedLong());
break;
case Tag.NextFileNumber:
result.SetNextFile((ulong)stream.Read7BitEncodedLong());
break;
case Tag.LastSequence:
result.SetLastSequence((ulong)stream.Read7BitEncodedLong());
break;
case Tag.CompactPointer:
level = stream.Read7BitEncodedInt();
var compactionPointer = stream.ReadLengthPrefixedInternalKey();
result.SetCompactionPointer(level, compactionPointer);
break;
case Tag.DeletedFile:
level = stream.Read7BitEncodedInt();
var fileNumber = (ulong)stream.Read7BitEncodedLong();
result.DeleteFile(level, fileNumber);
break;
case Tag.NewFile:
level = stream.Read7BitEncodedInt();
var fileMetadata = new FileMetadata
{
FileNumber = (ulong)stream.Read7BitEncodedLong(),
FileSize = stream.Read7BitEncodedLong(),
SmallestKey = stream.ReadLengthPrefixedInternalKey(),
LargestKey = stream.ReadLengthPrefixedInternalKey()
};
result.AddFile(level, fileMetadata);
break;
}
}
return result;
}
public void ShouldEncodeAndDecode()
{
var versionEdit = new VersionEdit();
versionEdit.SetComparatorName("testComparator");
versionEdit.SetLastSequence(10);
versionEdit.SetLogNumber(2);
versionEdit.SetNextFile(3);
versionEdit.SetPrevLogNumber(1);
versionEdit.SetLastSequence(1);
versionEdit.AddFile(0, new FileMetadata()
{
FileNumber = 0,
FileSize = 128,
LargestKey = new InternalKey("begin", Format.MaxSequenceNumber, ItemType.Value),
SmallestKey = new InternalKey("end", Format.MaxSequenceNumber, ItemType.Value),
});
versionEdit.AddFile(1, new FileMetadata()
{
FileNumber = 1,
FileSize = 128,
LargestKey = new InternalKey("begin", Format.MaxSequenceNumber, ItemType.Value),
SmallestKey = new InternalKey("end", Format.MaxSequenceNumber, ItemType.Value),
});
versionEdit.DeleteFile(0, 3);
versionEdit.DeleteFile(1, 4);
versionEdit.SetCompactionPointer(0, new InternalKey("begin", Format.MaxSequenceNumber, ItemType.Value));
versionEdit.SetCompactionPointer(1, new InternalKey("end", Format.MaxSequenceNumber, ItemType.Value));
var memoryStream = new MemoryStream();
var logWriter = new LogWriter(new InMemoryFileSystem("test"),memoryStream, new BufferPool());
versionEdit.EncodeTo(logWriter);
memoryStream.Position = 0;
var afterDecode = VersionEdit.DecodeFrom(new LogRecordStream(memoryStream, true, new BufferPool()));
Assert.Equal(versionEdit.Comparator, afterDecode.Comparator);
Assert.Equal(versionEdit.LogNumber, afterDecode.LogNumber);
Assert.Equal(versionEdit.NextFileNumber, afterDecode.NextFileNumber);
Assert.Equal(versionEdit.PrevLogNumber, afterDecode.PrevLogNumber);
Assert.Equal(versionEdit.LogNumber, afterDecode.LogNumber);
Assert.Equal(versionEdit.PrevLogNumber, afterDecode.PrevLogNumber);
Assert.Equal(versionEdit.LastSequence, afterDecode.LastSequence);
Assert.Equal(versionEdit.NewFiles[0][0].FileNumber, afterDecode.NewFiles[0][0].FileNumber);
Assert.Equal(versionEdit.NewFiles[0][0].FileSize, afterDecode.NewFiles[0][0].FileSize);
Assert.Equal(versionEdit.NewFiles[0][0].LargestKey, afterDecode.NewFiles[0][0].LargestKey);
Assert.Equal(versionEdit.NewFiles[0][0].SmallestKey, afterDecode.NewFiles[0][0].SmallestKey);
Assert.Equal(versionEdit.NewFiles[1][0].FileNumber, afterDecode.NewFiles[1][0].FileNumber);
Assert.Equal(versionEdit.NewFiles[1][0].FileSize, afterDecode.NewFiles[1][0].FileSize);
Assert.Equal(versionEdit.NewFiles[1][0].LargestKey, afterDecode.NewFiles[1][0].LargestKey);
Assert.Equal(versionEdit.NewFiles[1][0].SmallestKey, afterDecode.NewFiles[1][0].SmallestKey);
Assert.Equal(versionEdit.DeletedFiles[0][0], afterDecode.DeletedFiles[0][0]);
Assert.Equal(versionEdit.DeletedFiles[1][0], afterDecode.DeletedFiles[1][0]);
Assert.Equal(versionEdit.CompactionPointers[0][0], afterDecode.CompactionPointers[0][0]);
Assert.Equal(versionEdit.CompactionPointers[1][0], afterDecode.CompactionPointers[1][0]);
}
private static void AddMetadata(VersionEdit edit, StorageOptions options)
{
edit.SetComparatorName(options.Comparator.Name);
}
public async Task LogAndApplyAsync(VersionEdit edit, AsyncLock.LockScope locker)
{
string newManifestFile = null;
try
{
Version version;
using (await locker.LockAsync())
{
if (!edit.LogNumber.HasValue) edit.SetLogNumber(VersionSet.LogNumber);
else if (edit.LogNumber < VersionSet.LogNumber || edit.LogNumber >= VersionSet.NextFileNumber)
throw new InvalidOperationException("LogNumber");
if (!edit.PrevLogNumber.HasValue) edit.SetPrevLogNumber(VersionSet.PrevLogNumber);
edit.SetNextFile(VersionSet.NextFileNumber);
edit.SetLastSequence(VersionSet.LastSequence);
version = new Version(this, VersionSet);
var builder = new Builder(this, VersionSet, VersionSet.Current);
builder.Apply(edit);
builder.SaveTo(version);
Version.Finalize(version);
// Initialize new descriptor log file if necessary by creating
// a temporary file that contains a snapshot of the current version.
if (DescriptorLogWriter == null)
{
// No reason to unlock *mu here since we only hit this path in the
// first call to LogAndApply (when opening the database).
newManifestFile = FileSystem.DescriptorFileName(VersionSet.ManifestFileNumber);
edit.SetNextFile(VersionSet.NextFileNumber);
var descriptorFile = FileSystem.NewWritable(newManifestFile);
DescriptorLogWriter = new LogWriter(FileSystem, descriptorFile, Options.BufferPool);
Snapshooter.WriteSnapshot(DescriptorLogWriter, VersionSet);
}
}
// Write new record to MANIFEST log
edit.EncodeTo(DescriptorLogWriter);
// If we just created a new descriptor file, install it by writing a
// new CURRENT file that points to it.
if (!string.IsNullOrEmpty(newManifestFile))
{
SetCurrentFile(VersionSet.ManifestFileNumber);
// No need to double-check MANIFEST in case of error since it
// will be discarded below.
}
using (await locker.LockAsync())
{
// Install the new version
VersionSet.AppendVersion(version);
VersionSet.SetLogNumber(edit.LogNumber.Value);
VersionSet.SetPrevLogNumber(edit.PrevLogNumber.Value);
}
}
catch (Exception)
{
if (!string.IsNullOrEmpty(newManifestFile))
{
if (DescriptorLogWriter != null)
{
DescriptorLogWriter.Dispose();
DescriptorLogWriter = null;
}
FileSystem.DeleteFile(newManifestFile);
}
throw;
}
}
public void CreateNewDatabase()
{
var newDb = new VersionEdit();
newDb.SetComparatorName(Options.Comparator.Name);
newDb.SetLogNumber(0);
newDb.SetNextFile(2);
newDb.SetLastSequence(0);
var manifest = FileSystem.DescriptorFileName(1);
try
{
using (var file = FileSystem.NewWritable(manifest))
{
using (var logWriter = new LogWriter(FileSystem, file, Options.BufferPool))
{
newDb.EncodeTo(logWriter);
}
}
SetCurrentFile(1);
}
catch (Exception)
{
FileSystem.DeleteFile(manifest);
throw;
}
}
private ulong RecoverLogFile(ulong logNumber, VersionEdit edit, ulong maxSequence)
{
var logFileName = FileSystem.GetLogFileName(logNumber);
Log.Info("Starting to recover from log: {0}", logFileName);
MemTable mem = null;
using (var logFile = FileSystem.OpenForReading(logFileName))
{
foreach (var item in WriteBatch.ReadFromLog(logFile, Options.BufferPool))
{
var lastSequence = item.WriteSequence + (ulong)item.WriteBatch.OperationCount - 1;
if (lastSequence > maxSequence)
{
maxSequence = lastSequence;
}
if (mem == null)
{
mem = new MemTable(this);
}
item.WriteBatch.Prepare(mem);
item.WriteBatch.Apply(mem, new Reference<ulong> { Value = item.WriteSequence });
if (mem.ApproximateMemoryUsage > Options.WriteBatchSize)
{
Compactor.WriteLevel0Table(mem, null, edit);
mem = null;
}
}
}
if (mem != null)
{
Compactor.WriteLevel0Table(mem, null, edit);
}
return maxSequence;
}
public VersionEdit Recover()
{
FileSystem.EnsureDatabaseDirectoryExists();
FileSystem.Lock();
if (FileSystem.Exists(FileSystem.GetCurrentFileName()) == false)
{
if (Options.CreateIfMissing)
{
CreateNewDatabase();
}
else
{
throw new InvalidDataException(DatabaseName + " does not exist. Storage option CreateIfMissing is set to false.");
}
Log.Info("Creating db {0} from scratch", DatabaseName);
}
else
{
if (Options.ErrorIfExists)
{
throw new InvalidDataException(DatabaseName + " exists, while the ErrorIfExists option is set to true.");
}
Log.Info("Loading db {0} from existing source", DatabaseName);
}
VersionSet.Recover();
var minLog = VersionSet.LogNumber;
var prevLog = VersionSet.PrevLogNumber;
var databaseFiles = FileSystem.GetFiles();
var expected = VersionSet.GetLiveFiles();
var logNumbers = new List<ulong>();
foreach (var databaseFile in databaseFiles)
{
ulong number;
FileType fileType;
if (FileSystem.TryParseDatabaseFile(databaseFile, out number, out fileType))
{
expected.Remove(number);
if (fileType == FileType.LogFile && ((number >= minLog) || (number == prevLog)))
{
logNumbers.Add(number);
}
}
}
if (expected.Count > 0)
{
throw new CorruptedDataException(string.Format("Cannot recover because there are {0} missing files", expected.Count));
}
logNumbers.Sort();
ulong maxSequence = 0;
var edit = new VersionEdit();
foreach (var logNumber in logNumbers)
{
maxSequence = RecoverLogFile(logNumber, edit, maxSequence);
VersionSet.MarkFileNumberUsed(logNumber);
}
if (VersionSet.LastSequence < maxSequence)
{
VersionSet.LastSequence = maxSequence;
}
return edit;
}