public async Task Big() { var random = new Random(); var buffers = new List<byte[]>(); for (int i = 0; i < 15; i++) { var buffer = new byte[1044*33]; random.NextBytes(buffer); buffers.Add(buffer); } var memoryStream = new MemoryStream(); var logWriterStream = new LogWriter(new InMemoryFileSystem("test"),memoryStream, new BufferPool()); foreach (var buffer in buffers) { await CanReadAndWriteOkaySingleRecord.WriteRecordAsync(logWriterStream, buffer); } memoryStream.Position = 0; var logReader = new LogReader(memoryStream, true, 0, new BufferPool()); for (int i = 0; i < 15; i++) { Stream stream; Assert.True(logReader.TryReadRecord(out stream)); var actual = new MemoryStream(); stream.CopyTo(actual); Assert.Equal(buffers[i], actual.ToArray()); } }
public async Task Small() { var buffer = new byte[1044]; new Random().NextBytes(buffer); var memoryStream = new MemoryStream(); var logWriterStream = new LogWriter(new InMemoryFileSystem("test"), memoryStream, new BufferPool()); await WriteRecordAsync(logWriterStream, buffer); memoryStream.Position = 0; var logReader = new LogReader(memoryStream, true, 0, new BufferPool()); Stream stream; Assert.True(logReader.TryReadRecord(out stream)); var actual = new MemoryStream(); stream.CopyTo(actual); Assert.Equal(buffer, actual.ToArray()); }
public void Recover() { string currentManifest; using (var currentFile = storageContext.FileSystem.OpenForReading(storageContext.FileSystem.GetCurrentFileName())) using (var reader = new StreamReader(currentFile)) { currentManifest = reader.ReadToEnd(); } if (string.IsNullOrEmpty(currentManifest)) { throw new FormatException("CURRENT file should not be empty."); } log.Info("Current manifect is: {0}", currentManifest); using (var manifestFile = storageContext.FileSystem.OpenForReading(currentManifest)) { var logReader = new LogReader(manifestFile, true, 0, storageContext.Options.BufferPool); var builder = new Builder(storageContext, this, current); ulong? nextFileFromManifest = null; ulong? lastSequenceFromManifest = null; ulong? logNumberFromManifest = null; ulong? prevLogNumberFromManifest = null; Stream recordStream; while (logReader.TryReadRecord(out recordStream)) { VersionEdit edit; using (recordStream) { edit = VersionEdit.DecodeFrom(recordStream); } if (log.IsDebugEnabled) log.Debug("Read version edit with the following information:\r\n{0}", edit.DebugInfo); if (edit.Comparator != storageContext.Options.Comparator.Name) { throw new InvalidOperationException( string.Format("Decoded version edit comparator '{0}' does not match '{1}' that is currently in use.", edit.Comparator, storageContext.Options.Comparator.Name)); } builder.Apply(edit); if (edit.NextFileNumber.HasValue) { nextFileFromManifest = edit.NextFileNumber; } if (edit.PrevLogNumber.HasValue) { prevLogNumberFromManifest = edit.PrevLogNumber; } if (edit.LogNumber.HasValue) { logNumberFromManifest = edit.LogNumber; } if (edit.LastSequence.HasValue) { lastSequenceFromManifest = edit.LastSequence; } } if (nextFileFromManifest == null) { throw new ManifestFileException("No NextFileNumber entry"); } if (logNumberFromManifest == null) { throw new ManifestFileException("No LogNumber entry"); } if (lastSequenceFromManifest == null) { throw new ManifestFileException("No LastSequenceNumber entry"); } if (prevLogNumberFromManifest == null) { prevLogNumberFromManifest = 0; } MarkFileNumberUsed(prevLogNumberFromManifest.Value); MarkFileNumberUsed(logNumberFromManifest.Value); var version = new Version(storageContext, this); builder.SaveTo(version); Version.Finalize(version); AppendVersion(version); ManifestFileNumber = nextFileFromManifest.Value; NextFileNumber = nextFileFromManifest.Value + 1; LastSequence = lastSequenceFromManifest.Value; LogNumber = logNumberFromManifest.Value; PrevLogNumber = prevLogNumberFromManifest.Value; } }
internal static IEnumerable<LogReadResult> ReadFromLog(Stream logFile, BufferPool bufferPool) { var logReader = new LogReader(logFile, true, 0, bufferPool); Stream logRecordStream; while (logReader.TryReadRecord(out logRecordStream)) { var batch = new WriteBatch(); ulong seq; using (logRecordStream) { var buffer = new byte[8]; logRecordStream.ReadExactly(buffer, 8); seq = BitConverter.ToUInt64(buffer, 0); logRecordStream.ReadExactly(buffer, 4); var opCount = BitConverter.ToInt32(buffer, 0); for (var i = 0; i < opCount; i++) { logRecordStream.ReadExactly(buffer, 1); var op = (Operations)buffer[0]; var keyCount = logRecordStream.Read7BitEncodedInt(); var array = new byte[keyCount]; logRecordStream.ReadExactly(array, keyCount); var key = new Slice(array); switch (op) { case Operations.Delete: batch.Delete(key); break; case Operations.Put: logRecordStream.ReadExactly(buffer, 4); var size = BitConverter.ToInt64(buffer, 0); var value = new MemoryStream(); logRecordStream.CopyTo(value, size, LogWriter.BlockSize); batch.Put(key, value); break; default: throw new ArgumentException("Invalid operation type: " + op); } } } yield return new LogReadResult { WriteSequence = seq, WriteBatch = batch }; } }