/// <summary>
/// Wraps a generic stream into a block stream using the specified transformer and blocksize.
/// </summary>
/// <remarks>The wrapped stream must be writable and seekable</remarks>
/// <param name="wrappedStream">Stream to wrap</param>
/// <param name="transformer">The block transformer to use</param>
/// <param name="blockSize">Block size to use</param>
public BlockWriteOnceStream(Stream wrappedStream, IBlockTransformer transformer, short blockSize = BLOCK_SIZE) :
base(wrappedStream, transformer, blockSize)
{
currentBlock = new byte[BlockSize];
WrappedStream.Seek(0, SeekOrigin.Begin);
wrappedStream.SetLength(0);
}
private static void BlockStreamWriterOnceTest(IBlockTransformer transformer, IBlockCache cache)
{
using var ms = new KeepOpenMemoryStream();
const int COUNT = 100_000;
long expectedLength;
using (var writer = new BlockWriteOnceStream(ms, transformer)) {
using var binaryWriter = new BinaryWriter(writer, Encoding.ASCII, true);
for (var i = 0; i < COUNT; ++i)
{
binaryWriter.Write(i);
}
writer.Write(new byte[1 << 22]);
expectedLength = writer.Length;
}
using (var writer = new BlockReadOnlyStream(ms, transformer, cache: cache)) {
Assert.AreEqual(expectedLength, writer.Length);
using var binaryReader = new BinaryReader(writer, Encoding.ASCII, true);
using var binaryCursorReader = new BinaryReader(writer.CreateCursor(), Encoding.ASCII, true);
for (var i = 0; i < COUNT; ++i)
{
Assert.AreEqual(i, binaryReader.ReadInt32());
Assert.AreEqual(i, binaryCursorReader.ReadInt32());
}
var buf = new byte[1 << 22];
Assert.AreEqual(buf.Length, writer.Read(buf));
Assert.IsTrue(buf.All(i => i == 0));
}
}
private static void BlockStreamSequentialTestRunner(IBlockTransformer transformer)
{
var buf = new byte[1 << 20];
var outbuf = new byte[1 << 20];
buf.AsSpan().Fill(0x03);
using var ms = new KeepOpenMemoryStream();
using (var writer = new SequentialBlockWriteOnceStream(ms, transformer)) {
for (var i = 0; i < buf.Length; i += (int)Math.Floor(buf.Length / 20.0) + 1)
{
buf[i] = (byte)(i % byte.MaxValue);
writer.Write(buf.AsSpan(0, i));
writer.Flush(true);
}
}
ms.Seek(0, SeekOrigin.Begin);
using (var reader = new SequentialBlockReadonlyStream(ms, transformer)) {
for (var i = 0; i < buf.Length; i += (int)Math.Floor(buf.Length / 20.0) + 1)
{
outbuf.AsSpan(0, i).Clear();
var read = reader.Read(outbuf.AsSpan(0, i));
Assert.AreEqual(i, read);
Assert.IsTrue(buf.AsSpan(0, i).SequenceEqual(outbuf.AsSpan(0, i)));
}
}
}
/// <summary>
/// Explicitly set a custom block transformer
/// </summary>
/// <param name="transformer">The block tranformer to use</param>
/// <returns>New options with blocktransformer configured</returns>
/// <seealso cref="BlockTransformer" />
public PlaneDBOptions UsingBlockTransformer(IBlockTransformer transformer)
{
var rv = Clone();
rv.BlockTransformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
return(rv);
}
internal Journal(Stream stream, PlaneDBOptions options)
{
this.stream = stream;
transformer = options.BlockTransformer;
fullySync = options.MaxJournalActions < 0;
maxActions = Math.Max(0, options.MaxJournalActions);
stream.WriteInt32(Constants.MAGIC);
flusher = Task.Factory.StartNew(RunFlushLoop, cancel.Token, TaskCreationOptions.LongRunning,
TaskScheduler.Current);
}
public BlockReadOnlyStream(Stream wrappedStream, IBlockTransformer transformer, short blockSize = BLOCK_SIZE,
IBlockCache?cache = null) : base(wrappedStream, transformer, blockSize, cache)
{
ReadIndex();
if (wrappedStream is FileStream fstream)
{
mmap = MemoryMappedFile.CreateFromFile(fstream, null, 0, MemoryMappedFileAccess.Read, HandleInheritability.None,
true);
}
cursor = CreateCursor();
}
protected BlockStream(Stream wrappedStream, IBlockTransformer transformer, short blockSize = BLOCK_SIZE,
IBlockCache?cache = null)
{
if (blockSize < 512 || blockSize > 28671)
{
throw new ArgumentOutOfRangeException(nameof(blockSize));
}
WrappedStream = wrappedStream;
Transformer = transformer;
BlockSize = blockSize;
Cache = cache;
}
private void Test(IBlockTransformer transformer)
{
var buffer = Enumerable.Range(0, 128).Select(v => (byte)v).ToArray();
var cmp = Enumerable.Range(0, 128).Select(v => (byte)v).ToArray();
var trans = transformer.TransformBlock(buffer);
if (!transformer.MayChangeSize)
{
Assert.AreEqual(trans.Length, cmp.Length);
}
Assert.IsTrue(buffer.AsSpan().SequenceEqual(cmp));
var back = new byte[buffer.Length * 2];
var backspan = back.AsSpan(0, transformer.UntransformBlock(trans, back));
Assert.IsTrue(backspan.SequenceEqual(cmp));
}
private static void Main()
{
IntPtr algorithm;
uint result = BCryptCore.BCryptOpenAlgorithmProvider(out algorithm, "AES", null, 0);
IntPtr key;
byte[] secret = new byte[16];
byte[] iv = new byte[16];
result = BCryptCore.BCryptGenerateSymmetricKey(algorithm, out key, null, 0, secret, (ulong)secret.Length, 0);
byte[] plaintext = new byte[] { 0x6a, 0x84, 0x86, 0x7c, 0xd7, 0x7e, 0x12, 0xad, 0x07, 0xea, 0x1b, 0xe8, 0x95, 0xc5, 0x3f, 0xa3 };
byte[] ciphertext = new byte[plaintext.Length];
ulong pcbResult;
BCryptCore.BCryptEncrypt(key, plaintext, (ulong)plaintext.Length, IntPtr.Zero, null, 0, ciphertext, (ulong)ciphertext.Length, out pcbResult, 0);
Aes aes = new Aes();
IBlockTransformer encryptor = aes.CreateEncryptor(BlockCipherMode.ECB, secret, null);
byte[] ciphertext2 = encryptor.TransformFinalBlock(plaintext, 0, plaintext.Length);
}
public SequentialBlockReadonlyStream([NotNull] Stream wrappedStream, [NotNull] IBlockTransformer transformer,
short blockSize = BLOCK_SIZE) : base(wrappedStream, transformer, blockSize)
{
}
public CorpusTransformer(IBlockTransformer <TIn, TOut> blockTransformer)
{
this.blockTransformer = blockTransformer;
}
private static void BlockStreamWriterSizeTestRunner(IBlockTransformer transformer, [CanBeNull] IBlockCache cache)
{
BlockStreamWriterOnceTest(transformer, cache);
using var ms = new KeepOpenMemoryStream();
var items = 10_000;
using (var writer = new BlockRandomAccessStream(ms, transformer, cache: cache)) {
using (var binaryWriter = new BinaryWriter(writer, Encoding.ASCII, true)) {
for (var i = 0; i < items; ++i)
{
binaryWriter.Write(i);
}
}
Assert.AreEqual(writer.Position, writer.Length);
writer.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(writer.Position, 0);
Assert.AreEqual(writer.Length, sizeof(int) * items);
using var binaryReader = new BinaryReader(writer, Encoding.ASCII, true);
for (var i = 0; i < items; ++i)
{
Assert.AreEqual(i, binaryReader.ReadInt32());
}
}
using (var writer = new BlockRandomAccessStream(ms, transformer, cache: cache)) {
Assert.AreEqual(writer.Position, 0);
Assert.AreEqual(writer.Length, sizeof(int) * items);
using (var binaryReader = new BinaryReader(writer, Encoding.ASCII, true)) {
for (var i = 0; i < items; ++i)
{
Assert.AreEqual(i, binaryReader.ReadInt32());
}
}
Assert.AreEqual(writer.Position, writer.Length);
Assert.AreEqual(writer.Length, sizeof(int) * items);
if (transformer.MayChangeSize)
{
Assert.ThrowsException <IOException>(() => {
writer.Seek(sizeof(int), SeekOrigin.Begin);
using var binaryWriter = new BinaryWriter(writer, Encoding.ASCII, true);
for (var i = 0; i < items; ++i)
{
binaryWriter.Write(i);
}
});
Assert.AreEqual(sizeof(int), writer.Position);
}
writer.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(writer.Position, 0);
Assert.AreEqual(writer.Length, sizeof(int) * items);
using (var binaryReader = new BinaryReader(writer, Encoding.ASCII, true)) {
for (var i = 0; i < items; ++i)
{
Assert.AreEqual(i, binaryReader.ReadInt32());
}
}
Assert.AreEqual(writer.Position, writer.Length);
writer.Seek(-4, SeekOrigin.End);
Assert.AreEqual(writer.Position + 4, writer.Length);
Assert.AreEqual(4, writer.Read(new byte[5]));
Assert.AreEqual(writer.Position, writer.Length);
Assert.AreEqual(0, writer.Read(new byte[5]));
writer.SetLength(0);
Assert.AreEqual(writer.Length, 0);
Assert.AreEqual(writer.Position, 0);
writer.Flush(true);
}
}
private static void BlockStreamWriterSizeTestInternal(IBlockTransformer transformer)
{
BlockStreamWriterSizeTestRunner(transformer, null);
BlockStreamWriterSizeTestRunner(transformer, new BlockCache());
BlockStreamSequentialTestRunner(transformer);
}
