public async Task Big()
{
var random = new Random();
var buffers = new List<byte[]>();
for (int i = 0; i < 15; i++)
{
var buffer = new byte[1044*33];
random.NextBytes(buffer);
buffers.Add(buffer);
}
var memoryStream = new MemoryStream();
var logWriterStream = new LogWriter(new InMemoryFileSystem("test"),memoryStream, new BufferPool());
foreach (var buffer in buffers)
{
await CanReadAndWriteOkaySingleRecord.WriteRecordAsync(logWriterStream, buffer);
}
memoryStream.Position = 0;
var logReader = new LogReader(memoryStream, true, 0, new BufferPool());
for (int i = 0; i < 15; i++)
{
Stream stream;
Assert.True(logReader.TryReadRecord(out stream));
var actual = new MemoryStream();
stream.CopyTo(actual);
Assert.Equal(buffers[i], actual.ToArray());
}
}
public void WriteSnapshot(LogWriter logWriter, VersionSet versionSet)
{
var edit = new VersionEdit();
AddMetadata(edit, storageContext.Options);
AddCompactionPointers(edit, versionSet);
AddFiles(edit, versionSet);
edit.EncodeTo(logWriter);
}
public async Task Small()
{
var buffer = new byte[1044];
new Random().NextBytes(buffer);
var memoryStream = new MemoryStream();
var logWriterStream = new LogWriter(new InMemoryFileSystem("test"), memoryStream, new BufferPool());
await WriteRecordAsync(logWriterStream, buffer);
memoryStream.Position = 0;
var logReader = new LogReader(memoryStream, true, 0, new BufferPool());
Stream stream;
Assert.True(logReader.TryReadRecord(out stream));
var actual = new MemoryStream();
stream.CopyTo(actual);
Assert.Equal(buffer, actual.ToArray());
}
public void EncodeTo(LogWriter writer)
{
writer.RecordStarted();
if (HasComparator)
{
writer.Write7BitEncodedInt((int)Tag.Comparator);
writer.WriteLengthPrefixedSlice(Comparator);
}
if (LogNumber.HasValue)
{
writer.Write7BitEncodedInt((int)Tag.LogNumber);
writer.Write7BitEncodedLong((long)LogNumber.Value);
}
if (PrevLogNumber.HasValue)
{
writer.Write7BitEncodedInt((int)Tag.PrevLogNumber);
writer.Write7BitEncodedLong((long)PrevLogNumber.Value);
}
if (NextFileNumber.HasValue)
{
writer.Write7BitEncodedInt((int)Tag.NextFileNumber);
writer.Write7BitEncodedLong((long)NextFileNumber.Value);
}
if (LastSequence.HasValue)
{
writer.Write7BitEncodedInt((int)Tag.LastSequence);
writer.Write7BitEncodedLong((long)LastSequence.Value);
}
for (int level = 0; level < Config.NumberOfLevels; level++)
{
foreach (var compactionPointer in CompactionPointers[level])
{
writer.Write7BitEncodedInt((int)Tag.CompactPointer);
writer.Write7BitEncodedInt(level);
writer.WriteLengthPrefixedInternalKey(compactionPointer);
}
foreach (var fileNumber in DeletedFiles[level])
{
writer.Write7BitEncodedInt((int)Tag.DeletedFile);
writer.Write7BitEncodedInt(level);
writer.Write7BitEncodedLong((long)fileNumber);
}
foreach (var fileMetadata in NewFiles[level])
{
writer.Write7BitEncodedInt((int)Tag.NewFile);
writer.Write7BitEncodedInt(level);
writer.Write7BitEncodedLong((long)fileMetadata.FileNumber);
writer.Write7BitEncodedLong(fileMetadata.FileSize);
writer.WriteLengthPrefixedInternalKey(fileMetadata.SmallestKey);
writer.WriteLengthPrefixedInternalKey(fileMetadata.LargestKey);
}
}
writer.RecordCompleted(true);
}
public void ShouldEncodeAndDecode()
{
var versionEdit = new VersionEdit();
versionEdit.SetComparatorName("testComparator");
versionEdit.SetLastSequence(10);
versionEdit.SetLogNumber(2);
versionEdit.SetNextFile(3);
versionEdit.SetPrevLogNumber(1);
versionEdit.SetLastSequence(1);
versionEdit.AddFile(0, new FileMetadata()
{
FileNumber = 0,
FileSize = 128,
LargestKey = new InternalKey("begin", Format.MaxSequenceNumber, ItemType.Value),
SmallestKey = new InternalKey("end", Format.MaxSequenceNumber, ItemType.Value),
});
versionEdit.AddFile(1, new FileMetadata()
{
FileNumber = 1,
FileSize = 128,
LargestKey = new InternalKey("begin", Format.MaxSequenceNumber, ItemType.Value),
SmallestKey = new InternalKey("end", Format.MaxSequenceNumber, ItemType.Value),
});
versionEdit.DeleteFile(0, 3);
versionEdit.DeleteFile(1, 4);
versionEdit.SetCompactionPointer(0, new InternalKey("begin", Format.MaxSequenceNumber, ItemType.Value));
versionEdit.SetCompactionPointer(1, new InternalKey("end", Format.MaxSequenceNumber, ItemType.Value));
var memoryStream = new MemoryStream();
var logWriter = new LogWriter(new InMemoryFileSystem("test"),memoryStream, new BufferPool());
versionEdit.EncodeTo(logWriter);
memoryStream.Position = 0;
var afterDecode = VersionEdit.DecodeFrom(new LogRecordStream(memoryStream, true, new BufferPool()));
Assert.Equal(versionEdit.Comparator, afterDecode.Comparator);
Assert.Equal(versionEdit.LogNumber, afterDecode.LogNumber);
Assert.Equal(versionEdit.NextFileNumber, afterDecode.NextFileNumber);
Assert.Equal(versionEdit.PrevLogNumber, afterDecode.PrevLogNumber);
Assert.Equal(versionEdit.LogNumber, afterDecode.LogNumber);
Assert.Equal(versionEdit.PrevLogNumber, afterDecode.PrevLogNumber);
Assert.Equal(versionEdit.LastSequence, afterDecode.LastSequence);
Assert.Equal(versionEdit.NewFiles[0][0].FileNumber, afterDecode.NewFiles[0][0].FileNumber);
Assert.Equal(versionEdit.NewFiles[0][0].FileSize, afterDecode.NewFiles[0][0].FileSize);
Assert.Equal(versionEdit.NewFiles[0][0].LargestKey, afterDecode.NewFiles[0][0].LargestKey);
Assert.Equal(versionEdit.NewFiles[0][0].SmallestKey, afterDecode.NewFiles[0][0].SmallestKey);
Assert.Equal(versionEdit.NewFiles[1][0].FileNumber, afterDecode.NewFiles[1][0].FileNumber);
Assert.Equal(versionEdit.NewFiles[1][0].FileSize, afterDecode.NewFiles[1][0].FileSize);
Assert.Equal(versionEdit.NewFiles[1][0].LargestKey, afterDecode.NewFiles[1][0].LargestKey);
Assert.Equal(versionEdit.NewFiles[1][0].SmallestKey, afterDecode.NewFiles[1][0].SmallestKey);
Assert.Equal(versionEdit.DeletedFiles[0][0], afterDecode.DeletedFiles[0][0]);
Assert.Equal(versionEdit.DeletedFiles[1][0], afterDecode.DeletedFiles[1][0]);
Assert.Equal(versionEdit.CompactionPointers[0][0], afterDecode.CompactionPointers[0][0]);
Assert.Equal(versionEdit.CompactionPointers[1][0], afterDecode.CompactionPointers[1][0]);
}
public static async Task WriteRecordAsync(LogWriter logWriterStream, byte[] buffer)
{
logWriterStream.RecordStarted();
await logWriterStream.WriteAsync(buffer, 0, buffer.Length);
logWriterStream.RecordCompleted(true);
}
public async Task LogAndApplyAsync(VersionEdit edit, AsyncLock.LockScope locker)
{
string newManifestFile = null;
try
{
Version version;
using (await locker.LockAsync())
{
if (!edit.LogNumber.HasValue) edit.SetLogNumber(VersionSet.LogNumber);
else if (edit.LogNumber < VersionSet.LogNumber || edit.LogNumber >= VersionSet.NextFileNumber)
throw new InvalidOperationException("LogNumber");
if (!edit.PrevLogNumber.HasValue) edit.SetPrevLogNumber(VersionSet.PrevLogNumber);
edit.SetNextFile(VersionSet.NextFileNumber);
edit.SetLastSequence(VersionSet.LastSequence);
version = new Version(this, VersionSet);
var builder = new Builder(this, VersionSet, VersionSet.Current);
builder.Apply(edit);
builder.SaveTo(version);
Version.Finalize(version);
// Initialize new descriptor log file if necessary by creating
// a temporary file that contains a snapshot of the current version.
if (DescriptorLogWriter == null)
{
// No reason to unlock *mu here since we only hit this path in the
// first call to LogAndApply (when opening the database).
newManifestFile = FileSystem.DescriptorFileName(VersionSet.ManifestFileNumber);
edit.SetNextFile(VersionSet.NextFileNumber);
var descriptorFile = FileSystem.NewWritable(newManifestFile);
DescriptorLogWriter = new LogWriter(FileSystem, descriptorFile, Options.BufferPool);
Snapshooter.WriteSnapshot(DescriptorLogWriter, VersionSet);
}
}
// Write new record to MANIFEST log
edit.EncodeTo(DescriptorLogWriter);
// If we just created a new descriptor file, install it by writing a
// new CURRENT file that points to it.
if (!string.IsNullOrEmpty(newManifestFile))
{
SetCurrentFile(VersionSet.ManifestFileNumber);
// No need to double-check MANIFEST in case of error since it
// will be discarded below.
}
using (await locker.LockAsync())
{
// Install the new version
VersionSet.AppendVersion(version);
VersionSet.SetLogNumber(edit.LogNumber.Value);
VersionSet.SetPrevLogNumber(edit.PrevLogNumber.Value);
}
}
catch (Exception)
{
if (!string.IsNullOrEmpty(newManifestFile))
{
if (DescriptorLogWriter != null)
{
DescriptorLogWriter.Dispose();
DescriptorLogWriter = null;
}
FileSystem.DeleteFile(newManifestFile);
}
throw;
}
}
public void CreateNewLog()
{
var newFileNumber = VersionSet.NewFileNumber();
try
{
var name = FileSystem.GetLogFileName(newFileNumber);
var file = FileSystem.NewWritable(name);
LogWriter = new LogWriter(FileSystem, file, Options.BufferPool);
LogFileNumber = newFileNumber;
}
catch (Exception)
{
// Avoid chewing through file number space in a tight loop.
VersionSet.ReuseFileNumber(newFileNumber);
throw;
}
}
public void CreateNewDatabase()
{
var newDb = new VersionEdit();
newDb.SetComparatorName(Options.Comparator.Name);
newDb.SetLogNumber(0);
newDb.SetNextFile(2);
newDb.SetLastSequence(0);
var manifest = FileSystem.DescriptorFileName(1);
try
{
using (var file = FileSystem.NewWritable(manifest))
{
using (var logWriter = new LogWriter(FileSystem, file, Options.BufferPool))
{
newDb.EncodeTo(logWriter);
}
}
SetCurrentFile(1);
}
catch (Exception)
{
FileSystem.DeleteFile(manifest);
throw;
}
}