public async Task Big()
{
var random = new Random();
var buffers = new List<byte[]>();
for (int i = 0; i < 15; i++)
{
var buffer = new byte[1044*33];
random.NextBytes(buffer);
buffers.Add(buffer);
}
var memoryStream = new MemoryStream();
var logWriterStream = new LogWriter(new InMemoryFileSystem("test"),memoryStream, new BufferPool());
foreach (var buffer in buffers)
{
await CanReadAndWriteOkaySingleRecord.WriteRecordAsync(logWriterStream, buffer);
}
memoryStream.Position = 0;
var logReader = new LogReader(memoryStream, true, 0, new BufferPool());
for (int i = 0; i < 15; i++)
{
Stream stream;
Assert.True(logReader.TryReadRecord(out stream));
var actual = new MemoryStream();
stream.CopyTo(actual);
Assert.Equal(buffers[i], actual.ToArray());
}
}
public async Task Small()
{
var buffer = new byte[1044];
new Random().NextBytes(buffer);
var memoryStream = new MemoryStream();
var logWriterStream = new LogWriter(new InMemoryFileSystem("test"), memoryStream, new BufferPool());
await WriteRecordAsync(logWriterStream, buffer);
memoryStream.Position = 0;
var logReader = new LogReader(memoryStream, true, 0, new BufferPool());
Stream stream;
Assert.True(logReader.TryReadRecord(out stream));
var actual = new MemoryStream();
stream.CopyTo(actual);
Assert.Equal(buffer, actual.ToArray());
}
public void Recover()
{
string currentManifest;
using (var currentFile = storageContext.FileSystem.OpenForReading(storageContext.FileSystem.GetCurrentFileName()))
using (var reader = new StreamReader(currentFile))
{
currentManifest = reader.ReadToEnd();
}
if (string.IsNullOrEmpty(currentManifest))
{
throw new FormatException("CURRENT file should not be empty.");
}
log.Info("Current manifect is: {0}", currentManifest);
using (var manifestFile = storageContext.FileSystem.OpenForReading(currentManifest))
{
var logReader = new LogReader(manifestFile, true, 0, storageContext.Options.BufferPool);
var builder = new Builder(storageContext, this, current);
ulong? nextFileFromManifest = null;
ulong? lastSequenceFromManifest = null;
ulong? logNumberFromManifest = null;
ulong? prevLogNumberFromManifest = null;
Stream recordStream;
while (logReader.TryReadRecord(out recordStream))
{
VersionEdit edit;
using (recordStream)
{
edit = VersionEdit.DecodeFrom(recordStream);
}
if (log.IsDebugEnabled)
log.Debug("Read version edit with the following information:\r\n{0}", edit.DebugInfo);
if (edit.Comparator != storageContext.Options.Comparator.Name)
{
throw new InvalidOperationException(
string.Format("Decoded version edit comparator '{0}' does not match '{1}' that is currently in use.",
edit.Comparator, storageContext.Options.Comparator.Name));
}
builder.Apply(edit);
if (edit.NextFileNumber.HasValue)
{
nextFileFromManifest = edit.NextFileNumber;
}
if (edit.PrevLogNumber.HasValue)
{
prevLogNumberFromManifest = edit.PrevLogNumber;
}
if (edit.LogNumber.HasValue)
{
logNumberFromManifest = edit.LogNumber;
}
if (edit.LastSequence.HasValue)
{
lastSequenceFromManifest = edit.LastSequence;
}
}
if (nextFileFromManifest == null)
{
throw new ManifestFileException("No NextFileNumber entry");
}
if (logNumberFromManifest == null)
{
throw new ManifestFileException("No LogNumber entry");
}
if (lastSequenceFromManifest == null)
{
throw new ManifestFileException("No LastSequenceNumber entry");
}
if (prevLogNumberFromManifest == null)
{
prevLogNumberFromManifest = 0;
}
MarkFileNumberUsed(prevLogNumberFromManifest.Value);
MarkFileNumberUsed(logNumberFromManifest.Value);
var version = new Version(storageContext, this);
builder.SaveTo(version);
Version.Finalize(version);
AppendVersion(version);
ManifestFileNumber = nextFileFromManifest.Value;
NextFileNumber = nextFileFromManifest.Value + 1;
LastSequence = lastSequenceFromManifest.Value;
LogNumber = logNumberFromManifest.Value;
PrevLogNumber = prevLogNumberFromManifest.Value;
}
}
internal static IEnumerable<LogReadResult> ReadFromLog(Stream logFile, BufferPool bufferPool)
{
var logReader = new LogReader(logFile, true, 0, bufferPool);
Stream logRecordStream;
while (logReader.TryReadRecord(out logRecordStream))
{
var batch = new WriteBatch();
ulong seq;
using (logRecordStream)
{
var buffer = new byte[8];
logRecordStream.ReadExactly(buffer, 8);
seq = BitConverter.ToUInt64(buffer, 0);
logRecordStream.ReadExactly(buffer, 4);
var opCount = BitConverter.ToInt32(buffer, 0);
for (var i = 0; i < opCount; i++)
{
logRecordStream.ReadExactly(buffer, 1);
var op = (Operations)buffer[0];
var keyCount = logRecordStream.Read7BitEncodedInt();
var array = new byte[keyCount];
logRecordStream.ReadExactly(array, keyCount);
var key = new Slice(array);
switch (op)
{
case Operations.Delete:
batch.Delete(key);
break;
case Operations.Put:
logRecordStream.ReadExactly(buffer, 4);
var size = BitConverter.ToInt64(buffer, 0);
var value = new MemoryStream();
logRecordStream.CopyTo(value, size, LogWriter.BlockSize);
batch.Put(key, value);
break;
default:
throw new ArgumentException("Invalid operation type: " + op);
}
}
}
yield return new LogReadResult
{
WriteSequence = seq,
WriteBatch = batch
};
}
}