public T Deserialize(IRecordReader recordReader, IBinaryStorageFile <T> fileParser)
{
Debug.Assert(fileParser is StorageFile <T>, "cross call not allowed for WDB5 serializer");
Generator.Method.Invoke(recordReader, out var instance);
return(instance);
}
public AlignedRecordReader(IBinaryStorageFile fileReader, int recordSize)
{
_stringBlock = fileReader.FindBlockHandler <StringBlockHandler>(BlockIdentifier.StringBlock);
_stagingBuffer = new byte[recordSize + 8];
_byteCursor = 0;
}
public SerializerGenerator(IBinaryStorageFile storage, FieldInfoHandler <MemberMetadata> fieldInfoBlock) : base(storage.Type, storage.Options.TokenType, 0)
{
FieldInfoBlock = fieldInfoBlock;
if (storage.Header.IndexTable.Exists)
{
IndexColumn = storage.Header.IndexColumn;
}
}
public SerializerGenerator(IBinaryStorageFile storage) : base(storage.Type, storage.Options.TokenType, 0)
{
_memberMetadata = storage.FindSegment(SegmentIdentifier.FieldInfo)?.Handler as FieldInfoHandler <MemberMetadata>;
if (storage.Header.IndexTable.Exists)
{
_indexColumn = storage.Header.IndexColumn;
}
}
public UnalignedRecordReader(IBinaryStorageFile fileReader, int recordSize, Stream recordData, IntPtr stagingDataPointer)
{
_fileReader = fileReader;
_recordSize = recordSize;
_recordData = stagingDataPointer;
_managePointer = false;
using (var windowedStream = new WindowedStream(recordData, recordSize))
using (var outputStream = new IO.UnmanagedMemoryStream(stagingDataPointer, recordSize))
windowedStream.CopyTo(outputStream, recordSize);
}
public UnalignedRecordReader(IBinaryStorageFile fileReader, long recordSize)
{
_fileReader = fileReader;
_recordSize = recordSize;
// Allocating 7 extra bytes to guarantee we don't ever read out of our memory space
_recordData = MemoryPool <byte> .Shared.Rent((int)(recordSize + 7));
_stringBlock = _fileReader.FindSegment(SegmentIdentifier.StringBlock)?.Handler as StringBlockHandler;
// Read exactly what we need
fileReader.DataStream.Read(_recordData.Memory.Span.Slice(0, (int)recordSize));
}
public void ReadSegment(IBinaryStorageFile reader, long startOffset, long length)
{
if (length == 0)
{
return;
}
reader.DataStream.Position = startOffset;
while (reader.DataStream.Position <= (startOffset + length))
{
ReadPair(reader.DataStream);
}
}
public UnalignedRecordReader(IBinaryStorageFile fileReader, int recordSize, Stream recordData)
{
_fileReader = fileReader;
_recordSize = recordSize;
// Allocating 7 extra bytes to guarantee we don't ever read out of our memory space
// _recordData = Marshal.AllocHGlobal(recordData.Length + 7);
_recordData = Marshal.AllocHGlobal(recordSize);
_managePointer = true;
using (var windowedStream = new WindowedStream(recordData, recordSize))
using (var outputStream = new IO.UnmanagedMemoryStream(_recordData, recordSize))
windowedStream.CopyTo(outputStream, recordSize);
_bitCursor = 0;
}
public void ReadSegment(IBinaryStorageFile reader, long startOffset, long length)
{
if (length == 0)
{
return;
}
reader.DataStream.Position = startOffset;
while (reader.DataStream.Position < (startOffset + length))
{
var elementStore = ReadElement(reader.DataStream);
if (!EqualityComparer <TElement> .Default.Equals(elementStore, default))
{
_store.Add(elementStore);
}
}
}
public void ReadSegment(IBinaryStorageFile reader, long startOffset, long length)
{
// Impossible for length to be lower than 2
// Consider this: length 2 has to be 00 ??
// But strings have to be null terminated thus data should be 00 ?? 00
// Which is 3 bytes.
// Above sligtly wrong, nothing prevents a 1-byte string block with 00 but in that case we already handle it
// by returning string.Empty if offset was not found in the block.
// This is based off the assumption that strings at offset 0 will always be the null string
// which it has to be for empty string blocks. For non-empty blocks, let's just say blizzard's space saving
// track record isn't the best, and saving one byte by pointing the null string to the middle of any delimiter
// is something probably not worth the effort.
if (length <= 2)
{
return;
}
reader.DataStream.Position = startOffset;
// Next target: Pray for C# 9 to expose null-terminated strings
#if EXPERIMENTAL // This nets about 4% gain on string parsing alone (which is hardly a bottleneck, but still)
var internStrings = reader.Options.InternStrings;
// Requesting bytes aligned to int boundary
// Add an extra byte to ensure we never go out of bounds
var alignedLength = ((int)length + sizeof(int) - 1) & ~(sizeof(int) - 1);
var byteBuffer = ArrayPool <byte> .Shared.Rent(alignedLength);
Span <byte> byteSpan = new Span <byte>(byteBuffer, 0, alignedLength);
// Read exactly what is needed
reader.DataStream.Read(byteSpan.Slice(0, (int)length));
// Zero the trailing bytes because Rent does not guarantee that
// One is enough to ensure mask checking will fail
byteBuffer[length] = 0x00;
var stringOffset = 1L;
while (stringOffset < length)
{
var wordSpan = MemoryMarshal.Cast <byte, uint>(byteSpan.Slice((int)stringOffset));
var wordCursor = 0;
var mask = wordSpan[wordCursor];
while (((mask - 0x01010101) & ~mask & 0x80808080) == 0)
{
mask = wordSpan[++wordCursor];
}
var trailingCount = 0;
if ((mask & 0x000000FF) != 0x00)
{
++trailingCount;
if ((mask & 0x0000FF00) != 0x00)
{
++trailingCount;
if ((mask & 0x00FF0000) != 0x00)
{
++trailingCount;
}
}
}
var strLength = (wordCursor * sizeof(int) + trailingCount);
if (strLength > 0)
{
var value = (reader.Options.Encoding ?? Encoding.UTF8).GetString(byteBuffer, (int)stringOffset, strLength);
if (internStrings)
{
value = string.Intern(value);
}
_blockData.Add(stringOffset, value);
stringOffset += strLength + 1;
}
else
{
++stringOffset;
}
}
ArrayPool <byte> .Shared.Return(byteBuffer);
#else
var byteBuffer = ArrayPool <byte> .Shared.Rent((int)length);
var actualLength = reader.DataStream.Read(byteBuffer, 0, (int)length);
Debug.Assert(actualLength == length);
// We start at 1 because 0 is always 00, aka null string
var cursor = 1;
while (cursor != length)
{
var stringStart = cursor;
while (byteBuffer[cursor] != 0)
{
++cursor;
}
if (cursor - stringStart > 1)
{
var value = (reader.Options.Encoding ?? Encoding.UTF8).GetString(byteBuffer, stringStart, cursor - stringStart);
if (reader.Options.InternStrings)
{
value = string.Intern(value);
}
_blockData[stringStart] = value;
}
cursor += 1;
}
ArrayPool <byte> .Shared.Return(byteBuffer);
#endif
}
public ByteAlignedRecordReader(IBinaryStorageFile fileReader, int recordSize)
{
_stringBlock = fileReader.FindSegment(SegmentIdentifier.StringBlock)?.Handler as StringBlockHandler;
_stagingBuffer = new byte[recordSize + 8]; // Allocating 8 extra bytes for packed reads to make sure we don't start reading another process's memory out of bad luck
}
public InlinedStrings(IBinaryStorageFile fileReader) : base()
{
_encoding = fileReader.Options.Encoding ?? Encoding.UTF8;
}
public Serializer(IBinaryStorageFile storage) : base(storage)
{
Generator = new SerializerGenerator <T>(storage.Type, storage.Options.TokenType);
}
public Serializer(IBinaryStorageFile storage) : base(storage)
{
// Reuse WDBC's generator because there are literally no changes
Generator = new WDBC.SerializerGenerator <T>(storage.Type, storage.Options.TokenType);
}
public Serializer(IBinaryStorageFile storage) : base(storage)
{
InfoBlock = storage.FindSegmentHandler <FieldInfoHandler <MemberMetadata> >(SegmentIdentifier.FieldInfo);
Generator = new SerializerGenerator <T>(storage, InfoBlock);
}
public unsafe override void Read(IBinaryStorageFile storageFile)
{
var byteSpan = MemoryMarshal.AsBytes(Span);
storageFile.DataStream.Read(byteSpan);
}
public Serializer(IBinaryStorageFile storage) : base(storage)
{
Generator = new SerializerGenerator <T>(storage);
}
public abstract void Read(IBinaryStorageFile storageFile);
