/// <summary>
/// Provides a Stream that when read from, reads consecutive blocks of compressed data from an ORC Stream.
/// All data in the <paramref name="inputStream"/> will be consumed.
/// </summary>
public static IOStream GetDecompressingStream(IOStream inputStream, CompressionKind compressionKind)
{
if (compressionKind == CompressionKind.None)
{
return(inputStream);
}
else
{
return(new ConcatenatingStream(() =>
{
int blockLength;
bool isCompressed;
bool headerAvailable = ReadBlockHeader(inputStream, out blockLength, out isCompressed);
if (!headerAvailable)
{
return null;
}
var streamSegment = new StreamSegment(inputStream, blockLength, true);
if (!isCompressed)
{
return streamSegment;
}
else
{
return CompressionFactory.CreateDecompressorStream(compressionKind, streamSegment);
}
}, false));
}
}
public override void Open()
{
Close();
BinaryReader br = IOStream.GetBinaryReader();
int tableOffset = br.ReadInt32();
int dataOffset = br.ReadInt32();
int tableSize = dataOffset - tableOffset;
byte[] table = null;
if (tableSize != 0)
{
using (StreamSegment tableSegment = IOStream.GetStreamSegment(tableOffset, tableSize))
using (HalfByteStream tableStream = new HalfByteStream(tableSegment))
{
table = new byte[tableSize * 2];
tableStream.EnsureRead(table, 0, table.Length);
}
}
Table = table;
if (!IOStream.IsEndOfStream())
{
TimReader = new TimFileReader(IOStream.GetStreamSegment(IOStream.Position));
}
}
public object ReadValue(StreamSegment current)
{
if (current == null)
{
throw new ArgumentNullException(nameof(current));
}
if (current.UnderlyingStream != UnderlyingStream)
{
throw new ArgumentException("Invalid argument.", nameof(current));
}
switch (current.ContentType)
{
case StreamSegmentType.Null:
return null;
case StreamSegmentType.Integer:
return ReadIntegerCore(current);
case StreamSegmentType.String:
return ReadStringCore(current);
case StreamSegmentType.Binary:
return ReadBinaryCore(current);
case StreamSegmentType.Dictionary:
return ReadDictionaryCore(current);
case StreamSegmentType.Array:
return ReadArrayCore(current);
default:
throw new InvalidDataException();
}
}
public void Read_ZeroLength_ShouldReturn0()
{
var stream = new MemoryStream(); //Empty
var segment = new StreamSegment(stream, 0, false);
var result = ReadBytes(segment, 100);
Assert.Equal(0, result);
}
public void StreamSegment_Flush()
{
var s = new S();
var ss = new StreamSegment(s, 0, 8);
ss.Flush();
Assert.IsTrue(s.HasFlushed);
}
Protocol.Footer ReadFooter(Protocol.PostScript postScript, byte postScriptLength)
{
_inputStream.Seek(-1 - postScriptLength - (long)postScript.FooterLength, SeekOrigin.End);
var compressedStream = new StreamSegment(_inputStream, (long)postScript.FooterLength, true);
var footerStream = OrcCompressedStream.GetDecompressingStream(compressedStream, postScript.Compression);
return(Serializer.Deserialize <Protocol.Footer>(footerStream));
}
public bool TryCreateNextStream(string key, out Stream result, out Exception ex)
{
GtexMipMapLocation data = _data.MipMapData[_index++];
result = new StreamSegment(_resourceStream, data.Offset, data.Length, FileAccess.Read);
ex = null;
return(true);
}
Stream GetStream(ulong offset, ulong length)
{
//TODO move from using Streams to using MemoryMapped files or another data type that decouples the Stream Position from the Read call, allowing re-entrancy
_inputStream.Seek((long)offset, SeekOrigin.Begin);
var segment = new StreamSegment(_inputStream, (long)length, true);
return(OrcCompressedStream.GetDecompressingStream(segment, _compressionKind));
}
/// <summary>
/// Push new data into the parser
/// </summary>
/// <param name="s"></param>
public void Receive(IDataStream s)
{
lock (contentLock)
{
while (s.Length > 0)
{
while (!readingPayload && s.Length > 0)
{
missing = parser.ParseHeader(s);
if (missing <= 0)
{
var inner = s;
if (missing < 0)
{
var split = s.Length + missing;
inner = new StreamSegment(s, 0, split);
s = new StreamSegment(s, split);
}
else
{
s = EmptyData.Instance;
}
var ret = parser.Close(inner);
parser = new WebSocketMessage.Parser();
onMessageParsed(ret);
}
else
{
readingPayload = true;
dataBuilder.Append(s);
return;
}
}
if (readingPayload)
{
if (s.Length < missing)
{
dataBuilder.Append(s);
missing -= s.Length;
return;
}
else
{
dataBuilder.Append(new StreamSegment(s, 0, missing));
s = new StreamSegment(s, missing);
missing = 0;
readingPayload = false;
var bin = dataBuilder.Close();
dataBuilder = new ThresholdedDataBuilder(logger);
var msg = parser.Close(bin);
parser = new WebSocketMessage.Parser();
onMessageParsed(msg);
}
}
}
}
}
public void Read_OneByteAtATime_ShouldReturnCorrectResult()
{
var stream = new MemoryStream(new byte[] { 0x01, 0x02, 0x03 });
var segment = new StreamSegment(stream, 3, false);
Assert.Equal(0x01, segment.ReadByte());
Assert.Equal(0x02, segment.ReadByte());
Assert.Equal(0x03, segment.ReadByte());
}
private Metadata ReadMetadata(PostScript postScript, byte postScriptLength)
{
var offset = -1 - postScriptLength - (long)postScript.FooterLength - (long)postScript.MetadataLength;
_inputStream.Seek(offset, SeekOrigin.End);
var compressedStream = new StreamSegment(_inputStream, (long)postScript.MetadataLength, true);
var metadataStream = OrcCompressedStream.GetDecompressingStream(compressedStream, postScript.Compression);
return(Serializer.Deserialize <Metadata>(metadataStream));
}
private static Stream GetPrimeSegment(out byte[] buffer)
{
var bs = new byte[] { 2, 3, 5, 7, 11, 13, 17, 19, 23, 29 };
var ms = new MemoryStream(bs);
var ss = new StreamSegment(ms, 2, 7);
buffer = bs;
return(ss);
}
private static TextureSection[] ReadTextureHeaders(Stream xgrStream, WpdEntry[] textureEntries)
{
TextureSection[] result = new TextureSection[textureEntries.Length];
for (int i = 0; i < result.Length; i++)
{
WpdEntry entry = textureEntries[i];
using (StreamSegment textureHeaderInput = new StreamSegment(xgrStream, entry.Offset, entry.Length, FileAccess.Read))
result[i] = textureHeaderInput.ReadContent <TextureSection>();
}
return(result);
}
public void Setup()
{
for (int i = 0; i < 6; ++i)
{
_streamSegment[i] = new StreamSegment
{
Start = i * 2,
End = i * 2 + 1
};
}
}
public void Setup()
{
for (int i = 0; i < 6; ++i)
{
_streamSegment[i] = new StreamSegment
{
Start = i * 2,
End = i * 2 + 1
};
}
}
public void Read_Overrun_ShouldReturn0()
{
var stream = new MemoryStream(new byte[] { 0x01, 0x02, 0x03 });
var segment = new StreamSegment(stream, 2, false);
var successfulRead = ReadBytes(segment, 2);
Assert.Equal(2, successfulRead);
var unsuccessfulRead = ReadBytes(segment, 1);
Assert.Equal(0, unsuccessfulRead);
}
public StreamSegment ReadNext(StreamSegment current)
{
if (current == null)
{
throw new ArgumentNullException(nameof(current));
}
if (current.Next >= UnderlyingStream.Length)
{
return null;
}
UnderlyingStream.Seek(current.Next, SeekOrigin.Begin);
return ReadSegment();
}
private static WflContent[] ReadFontContent(Stream xgr, WpdEntry[] fontEntries)
{
WflContent[] result = new WflContent[fontEntries.Length];
for (int i = 0; i < result.Length; i++)
{
WpdEntry entry = fontEntries[i];
using (StreamSegment wflInput = new StreamSegment(xgr, entry.Offset, entry.Length, FileAccess.Read))
{
WflFileReader reader = new WflFileReader(wflInput);
result[i] = reader.Read();
}
}
return(result);
}
/// <inheritdoc/>
public byte ReadByte(long index)
{
IDataStreamHelpers.ReadByteArgsCheck(this, index);
ChunkHint hint;
lock (dataLock)
{
var hintIdx = locateChunk(index);
hint = chunkHints[hintIdx];
}
var seg = new StreamSegment(stream, hint.StreamStart);
return(seg.ReadByte(index - hint.DataStart));
}
private void parseChunkedBody()
{
if (contentData.Length == 0)
{
// wait for some data
return;
}
var seg = new StreamSegment(contentData);
while (seg.Length > 0)
{
ChunkedDecoder.DecodeOneInfo chunkInfo;
try
{
chunkInfo = ChunkedDecoder.DecodeOneChunk(seg);
}
catch (PartialChunkException)
{
// wait for more data
contentData = new StreamList();
if (seg.Length > 0)
{
contentData.Add(new ByteArray(seg));
}
return;
}
catch
{
throw new HttpInvalidMessageException();
}
// finished
if (chunkInfo.DataLength == 0)
{
seg = new StreamSegment(seg, chunkInfo.ChunkLength);
contentData = new StreamList();
contentData.Add(new ByteArray(seg));
state = ParsingState.CheckingChunedTrailer;
return;
}
else
{
dechunkedLength += chunkInfo.DataLength;
seg = new StreamSegment(contentData, seg.Offset + chunkInfo.ChunkLength, seg.Length - chunkInfo.ChunkLength);
}
}
contentData = new StreamList();
return;
}
private readonly StreamSegment[] readers; // a pool of read-only streams that can be shared between multiple readers in parallel
private protected ConcurrentStorageAccess(string fileName, int bufferSize, int readersCount, FileOptions options)
{
fs = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, bufferSize, options);
readers = new StreamSegment[readersCount];
if (readersCount == 1)
{
readers[0] = new StreamSegment(fs, true);
}
else
{
foreach (ref var reader in readers.AsSpan())
{
reader = new StreamSegment(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, bufferSize, FileOptions.Asynchronous | FileOptions.RandomAccess), false);
}
}
}
public static (PostScript, byte) ReadPostScript(this System.IO.Stream inputStream)
{
inputStream.Seek(-1, SeekOrigin.End);
byte postScriptLength = inputStream.CheckedReadByte();
inputStream.Seek(-1 - postScriptLength, SeekOrigin.End);
var stream = new StreamSegment(inputStream, postScriptLength, true);
var postScript = Serializer.Deserialize <PostScript>(stream);
if (postScript.Magic != "ORC")
{
throw new InvalidDataException("Postscript didn't contain magic bytes");
}
return(postScript, postScriptLength);
}
/// <summary>
/// Initializes a new persistent audit trail.
/// </summary>
/// <param name="path">The path to the folder to be used by audit trail.</param>
/// <param name="recordsPerPartition">The maximum number of log entries that can be stored in the single file called partition.</param>
/// <param name="configuration">The configuration of the persistent audit trail.</param>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="recordsPerPartition"/> is less than 2.</exception>
public PersistentState(DirectoryInfo path, int recordsPerPartition, Options?configuration = null)
{
if (configuration is null)
{
configuration = new Options();
}
if (recordsPerPartition < 2L)
{
throw new ArgumentOutOfRangeException(nameof(recordsPerPartition));
}
if (!path.Exists)
{
path.Create();
}
backupCompression = configuration.BackupCompression;
replayOnInitialize = configuration.ReplayOnInitialize;
bufferSize = configuration.BufferSize;
location = path;
this.recordsPerPartition = recordsPerPartition;
initialSize = configuration.InitialPartitionSize;
commitEvent = new AsyncManualResetEvent(false);
sessionManager = new DataAccessSessionManager(configuration.MaxConcurrentReads, configuration.GetMemoryAllocator <byte>(), bufferSize);
syncRoot = new AsyncSharedLock(sessionManager.Capacity);
entryPool = configuration.GetMemoryAllocator <LogEntry>();
metadataPool = configuration.UseCaching ? configuration.GetMemoryAllocator <LogEntryMetadata>() : null;
nullSegment = new StreamSegment(Stream.Null);
initialEntry = new LogEntry(nullSegment, sessionManager.WriteSession.Buffer, new LogEntryMetadata());
// sorted dictionary to improve performance of log compaction and snapshot installation procedures
partitionTable = new SortedDictionary <long, Partition>();
// load all partitions from file system
foreach (var file in path.EnumerateFiles())
{
if (long.TryParse(file.Name, out var partitionNumber))
{
var partition = new Partition(file.Directory, bufferSize, recordsPerPartition, partitionNumber, metadataPool, sessionManager.Capacity);
partition.PopulateCache(sessionManager.WriteSession);
partitionTable[partitionNumber] = partition;
}
}
state = new NodeState(path, AsyncLock.Exclusive(syncRoot));
snapshot = new Snapshot(path, bufferSize, sessionManager.Capacity);
snapshot.PopulateCache(sessionManager.WriteSession);
}
Protocol.PostScript ReadPostScript(out byte postScriptLength)
{
_inputStream.Seek(-1, SeekOrigin.End);
postScriptLength = _inputStream.CheckedReadByte();
_inputStream.Seek(-1 - postScriptLength, SeekOrigin.End);
var stream = new StreamSegment(_inputStream, postScriptLength, true);
var postScript = Serializer.Deserialize <Protocol.PostScript>(stream);
if (postScript.Magic != "ORC")
{
throw new InvalidDataException("Postscript didn't contain magic bytes");
}
return(postScript);
}
public ChunkHint ComputeNextHint(DechunkedStream stream)
{
var seg = new StreamSegment(stream.stream, NextChunk);
var chunkInfo = ChunkedDecoder.DecodeOneChunk(seg);
// last chunk
if (chunkInfo.DataLength == 0)
{
return(null);
}
else
{
return new ChunkHint {
StreamStart = this.NextChunk + chunkInfo.DataStart, NextChunk = this.NextChunk + chunkInfo.ChunkLength, DataLength = chunkInfo.DataLength, DataStart = this.DataStart + this.DataLength
}
};
}
}
public void StreamSegment_Ctor_ArgumentChecking()
{
var ms = new MemoryStream(new byte[5]);
Assert.ThrowsException <ArgumentNullException>(() => _ = new StreamSegment(stream: null, 0, 5));
Assert.ThrowsException <ArgumentOutOfRangeException>(() => _ = new StreamSegment(ms, -1, 5));
Assert.ThrowsException <ArgumentOutOfRangeException>(() => _ = new StreamSegment(ms, 0, -1));
Assert.ThrowsException <ArgumentException>(() => _ = new StreamSegment(ms, 0, 6));
Assert.ThrowsException <ArgumentException>(() => _ = new StreamSegment(ms, 1, 5));
Assert.ThrowsException <ArgumentException>(() => _ = new StreamSegment(ms, 4, 2));
Assert.ThrowsException <ArgumentException>(() => _ = new StreamSegment(ms, 5, 1));
for (long s = 0; s <= 4; s++)
{
for (long l = 0; l <= 5 - s; l++)
{
Assert.IsNotNull(new StreamSegment(ms, s, l)); // shall not throw
}
}
}
/// <inheritdoc/>
public void ReadBytesToBuffer(byte[] buffer, long start = 0, int length = -1, int offset = 0)
{
IDataStreamHelpers.ReadBytesToBufferArgsCheck(this, buffer, start, ref length, offset);
if (length == 0)
{
return;
}
var remaining = length;
var hintIdx = locateChunk(start);
var hint = chunkHints[hintIdx];
// these 2 must be close together, 2GiB chunks are too big
int chunkOffset = (int)(start - hint.DataStart);
while (remaining > 0)
{
lock (dataLock)
{
if (chunkHints.Count <= hintIdx)
{
hint = hint.ComputeNextHint(this);
chunkHints.Add(hint);
}
else
{
hint = chunkHints[hintIdx];
}
}
var seg = new StreamSegment(stream, hint.StreamStart);
var read = (int)Math.Min(hint.DataLength - chunkOffset, remaining);
seg.ReadBytesToBuffer(buffer, chunkOffset, read, offset);
offset += read;
remaining -= read;
chunkOffset = 0;
++hintIdx;
}
}
internal LogEntry(StreamSegment cachedContent, Memory <byte> sharedBuffer, in LogEntryMetadata metadata)
public string ReadInteger(StreamSegment current)
{
if (current == null)
{
throw new ArgumentNullException(nameof(current));
}
if (current.UnderlyingStream != UnderlyingStream)
{
throw new ArgumentException("Invalid argument.", nameof(current));
}
switch (current.ContentType)
{
case StreamSegmentType.Integer:
return ReadStringCore(current);
default:
throw new InvalidDataException();
}
}
public Stream ReadBinaryAsStream(StreamSegment current)
{
if (current == null)
{
throw new ArgumentNullException(nameof(current));
}
if (current.UnderlyingStream != UnderlyingStream)
{
throw new ArgumentException("Invalid argument.", nameof(current));
}
switch (current.ContentType)
{
case StreamSegmentType.Null:
return Stream.Null;
case StreamSegmentType.Binary:
// todo : use underlying stream later.
return new MemoryStream(ReadBinaryCore(current));
default:
throw new InvalidDataException();
}
}
private byte[] ReadBinaryCore(StreamSegment current)
{
UnderlyingStream.Seek(current.ContentStartOffset, SeekOrigin.Begin);
return _reader.ReadBytes(current.ContentLength);
}
public object[] ReadArray(StreamSegment current)
{
if (current == null)
{
throw new ArgumentNullException(nameof(current));
}
if (current.UnderlyingStream != UnderlyingStream)
{
throw new ArgumentException("Invalid argument.", nameof(current));
}
switch (current.ContentType)
{
case StreamSegmentType.Null:
return null;
case StreamSegmentType.Array:
return ReadArrayCore(current);
default:
throw new InvalidDataException();
}
}
public Dictionary<string, Lazy<object>> ReadDictionaryLazy(StreamSegment current)
{
if (current == null)
{
throw new ArgumentNullException(nameof(current));
}
if (current.UnderlyingStream != UnderlyingStream)
{
throw new ArgumentException("Invalid argument.", nameof(current));
}
switch (current.ContentType)
{
case StreamSegmentType.Null:
return null;
case StreamSegmentType.Dictionary:
return ReadDictionaryLazyCore(current);
default:
throw new InvalidDataException();
}
}
private object[] ReadArrayCore(StreamSegment current)
{
UnderlyingStream.Seek(current.ContentStartOffset, SeekOrigin.Begin);
var indices = new int[current.ContentLength / 4];
for (int i = 0; i < indices.Length; i++)
{
indices[i] = _reader.ReadInt32();
}
var result = new object[current.ContentLength / 4];
for (int i = 0; i < indices.Length; i++)
{
result[i] = ReadValue(ReadSegment(indices[i]));
}
return result;
}
private Dictionary<string, Lazy<object>> ReadDictionaryLazyCore(StreamSegment current)
{
UnderlyingStream.Seek(current.ContentStartOffset, SeekOrigin.Begin);
var indices = new int[current.ContentLength / 4];
for (int i = 0; i < indices.Length; i++)
{
indices[i] = _reader.ReadInt32();
}
var result = new Dictionary<string, Lazy<object>>();
for (int i = 0; i < indices.Length;)
{
var keySeg = ReadSegment(indices[i++]);
var valueSeg = ReadSegment(indices[i++]);
result[ReadString(keySeg)] = new Lazy<object>(() => ReadValue(valueSeg), false);
}
return result;
}
private int ReadIntegerCore(StreamSegment current)
{
UnderlyingStream.Seek(current.ContentStartOffset, SeekOrigin.Begin);
return _reader.ReadInt32();
}
public void Setup()
{
_streamSegment = new StreamSegment();
_streamSegment.Start = 10;
_streamSegment.End = 20;
}
public void Setup()
{
_streamSegment = new StreamSegment();
_streamSegment.Start = 10;
_streamSegment.End = 20;
}
private string ReadStringCore(StreamSegment current)
{
UnderlyingStream.Seek(current.ContentStartOffset, SeekOrigin.Begin);
return _reader.ReadString();
}
