public static CharacterCreateRequest Unmarshal(MessageEventArgs args)
{
var reader = new SpanReader(args.Message.Payload.Span);
var slot = reader.ReadByte();
var c = new Character();
c.Level = 1;
c.Name = reader.ReadDaocString();
reader.Skip(4); // 0x18 0x00 0x00 0x00 - DoL notes say this is an array length
c.Customization = reader.Read <Customizations>();
reader.Skip(9);
var operation = reader.ReadByte(); // 1
var custType = reader.ReadByte(); // 0
reader.Skip(2); // ???
c.LocationDescription = reader.ReadDaocString(); // empty
reader.ReadDaocString(); // class description; redundant and empty
reader.ReadDaocString(); // race description; redundant and empty
reader.Skip(1); // ???
c.Classification = reader.Read <Classification>();
c.Model = reader.ReadUInt16LittleEndian(); // 12352
c.Region = reader.ReadUInt16LittleEndian(); // byte? 27
reader.Skip(4);
c.Stats = reader.Read <Stats>();
c.Equipment = reader.Read <Equipment>(); // empty
; // 4 bytes left
return(new CharacterCreateRequest(slot, c));
}
private Tensor <T> LoadVarData <T>(string name)
where T : unmanaged
{
var v = GetVar(name);
var reader = new SpanReader(File.ReadAllBytes(Path.Combine(_modelPath, name)));
var version = reader.Read <uint>();
var lodLevel = reader.Read <ulong>();
for (uint i = 0; i < lodLevel; i++)
{
var len = reader.Read <ulong>();
reader.Skip((int)len);
}
version = reader.Read <uint>();
if (version != 0)
{
throw new NotSupportedException();
}
{
var descSize = reader.Read <int>();
reader.Skip(descSize);
}
var rest = reader.ReadAsSpan();
var data = MemoryMarshal.Cast <byte, T>(rest);
return(new DenseTensor <T>(data.ToArray(), GetVarShape(name)));
}
internal CorrelationId(ReadOnlySpan <byte> bytes, out int consumedBytes)
{
var reader = new SpanReader <byte>(bytes);
ApplicationId = BinaryPrimitives.ReadInt64LittleEndian(reader.Read(sizeof(long)));
StreamId = BinaryPrimitives.ReadInt64LittleEndian(reader.Read(sizeof(long)));
consumedBytes = reader.ConsumedCount;
}
/// <summary>
/// Initializes a new unique identifier from set of bytes.
/// </summary>
/// <param name="bytes">The memory block of bytes.</param>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="bytes"/> is too small.</exception>
public ClusterMemberId(ReadOnlySpan <byte> bytes)
{
var reader = new SpanReader <byte>(bytes);
address = reader.Read <Guid>();
port = reader.Read <int>();
length = reader.Read <int>();
family = reader.Read <int>();
}
internal static void Parse(ReadOnlySpan <byte> payload, out ushort remotePort, out long term, out long lastLogIndex, out long lastLogTerm)
{
var reader = new SpanReader <byte>(payload);
remotePort = ReadUInt16LittleEndian(reader.Read(sizeof(ushort)));
term = ReadInt64LittleEndian(reader.Read(sizeof(long)));
lastLogIndex = ReadInt64LittleEndian(reader.Read(sizeof(long)));
lastLogTerm = ReadInt64LittleEndian(reader.Read(sizeof(long)));
}
internal static int ParseLogEntryPrologue(ReadOnlySpan <byte> input, out long length, out long term, out DateTimeOffset timeStamp, out bool isSnapshot)
{
var reader = new SpanReader <byte>(input);
length = ReadInt64LittleEndian(reader.Read(sizeof(long)));
term = ReadInt64LittleEndian(reader.Read(sizeof(long)));
timeStamp = reader.Read <DateTimeOffset>();
isSnapshot = ValueTypeExtensions.ToBoolean(reader.Read());
return(reader.ConsumedCount);
}
private List <KeyValuePair <byte[], ResultCacheEntry> > DecodeBatch(ReadOnlySpan <byte> data)
{
SpanReader batchReader = new SpanReader(data);
var sequenceNumber = batchReader.ReadInt64();
var operationCount = batchReader.ReadInt32();
var result = new List <KeyValuePair <byte[], ResultCacheEntry> >(operationCount);
for (int i = 0; i < operationCount; i++)
{
byte operationCode = batchReader.ReadByte();
var keyLength = batchReader.ReadVarLong();
var currentKey = batchReader.Read(keyLength);
if (operationCode == (int)OperationType.Put) // Put
{
ulong valueLength = batchReader.ReadVarLong();
var currentVal = batchReader.Read(valueLength);
result.Add(new KeyValuePair <byte[], ResultCacheEntry>(currentKey.ToArray(), new ResultCacheEntry
{
Sequence = sequenceNumber,
ResultState = ResultState.Exist,
Data = currentVal.ToArray()
}));
}
else if (operationCode == (int)OperationType.Delete) // Delete
{
// says return "not found" in this case. Need to investigate since I believe there can multiple records with same key in this case.
result.Add(new KeyValuePair <byte[], ResultCacheEntry>(currentKey.ToArray(), new ResultCacheEntry
{
Sequence = sequenceNumber,
ResultState = ResultState.Deleted
}));
}
else
{
// unknown recType
}
}
return(result);
}
private bool TryParseCurrentEntry()
{
if (!HasNext())
{
return(false);
}
var reader = new SpanReader(_blockData.Span)
{
Position = _position
};
// An entry for a particular key-value pair has the form:
// shared_bytes: varint32
var sharedBytes = reader.ReadVarLong();
// unshared_bytes: varint32
var nonSharedBytes = reader.ReadVarLong();
// value_length: varint32
var valueLength = reader.ReadVarLong();
// key_delta: char[unshared_bytes]
ReadOnlyMemory <byte> keyDelta = reader.Read(nonSharedBytes).ToArray();
var entry = new BlockEntry();
Memory <byte> combinedKey = new byte[sharedBytes + nonSharedBytes];
if (sharedBytes > 0)
{
if (Current == null)
{
throw new Exception("Faulty state. Got shared key, but had no current entry.");
}
Current.Key.Slice(0, (int)sharedBytes).CopyTo(combinedKey.Slice(0, (int)sharedBytes));
}
keyDelta.Slice(0, (int)nonSharedBytes).CopyTo(combinedKey.Slice((int)sharedBytes, (int)nonSharedBytes));
entry.Key = combinedKey;
entry.Data = reader.Read(valueLength).ToArray();
// Position stream after value of this entry
_position = reader.Position;
Current = entry;
return(true);
}
internal static int ParseAnnouncement(ReadOnlySpan <byte> input, out ushort remotePort, out long term, out long snapshotIndex, out long length, out long snapshotTerm, out DateTimeOffset timestamp)
{
var reader = new SpanReader <byte>(input);
remotePort = reader.ReadUInt16(true);
term = reader.ReadInt64(true);
snapshotIndex = reader.ReadInt64(true);
length = reader.ReadInt64(true);
snapshotTerm = reader.ReadInt64(true);
timestamp = reader.Read <DateTimeOffset>();
return(reader.ConsumedCount);
}
internal static void ParseAnnouncement(ReadOnlySpan <byte> input, out ushort remotePort, out long term, out long prevLogIndex, out long prevLogTerm, out long commitIndex, out int entriesCount)
{
var reader = new SpanReader <byte>(input);
remotePort = ReadUInt16LittleEndian(reader.Read(sizeof(ushort)));
term = ReadInt64LittleEndian(reader.Read(sizeof(long)));
prevLogIndex = ReadInt64LittleEndian(reader.Read(sizeof(long)));
prevLogTerm = ReadInt64LittleEndian(reader.Read(sizeof(long)));
commitIndex = ReadInt64LittleEndian(reader.Read(sizeof(long)));
entriesCount = ReadInt32LittleEndian(reader.Read(sizeof(int)));
}
/// <summary>
/// The second layer of the search tree is a table file's block index of keys. The block index is part of the table
/// file's metadata which is located at the end of its physical data file. The index contains one entry for each
/// logical data block within the table file. The entry contains the last key in the block and the offset of the block
/// within the table file. leveldb performs a binary search of the block index to locate a candidate data block. It
/// reads the candidate data block from the table file.
/// </summary>
public static Footer Read(Stream stream)
{
stream.Seek(-FooterLength, SeekOrigin.End);
Span <byte> footer = new byte[FooterLength];
stream.Read(footer);
var reader = new SpanReader(footer);
var metaIndexHandle = BlockHandle.ReadBlockHandle(ref reader);
var indexHandle = BlockHandle.ReadBlockHandle(ref reader);
reader.Seek(-sizeof(ulong), SeekOrigin.End);
ReadOnlySpan <byte> magic = reader.Read(8);
if (Magic.AsSpan().SequenceCompareTo(magic) != 0)
{
throw new Exception("Invalid footer. Magic end missing. This is not a proper table file");
}
return(new Footer(metaIndexHandle, indexHandle));
}
private static long SolvePart2(ReadOnlySpan <byte> input)
{
long length = 0;
var reader = new SpanReader(input);
while (!reader.Done)
{
if (reader.Read() == '(')
{
int repLength = reader.ReadPosIntUntil('x');
int repCount = reader.ReadPosIntUntil(')');
ReadOnlySpan <byte> rep = reader.ReadBytes(repLength);
length += SolvePart2(rep) * repCount;
}
else
{
length += 1;
}
}
return(length);
}
private static int SolvePart1(ReadOnlySpan <byte> input)
{
int length = 0;
var reader = new SpanReader(input);
while (!reader.Done)
{
if (reader.Read() == '(')
{
int repLength = reader.ReadPosIntUntil('x');
int repCount = reader.ReadPosIntUntil(')');
length += repLength * repCount;
reader.SkipLength(repLength);
}
else
{
length += 1;
}
}
return(length);
}
//[Test]
//public void LevelDbGetValueFromMissingKey()
//{
// using var db = new Database(new DirectoryInfo(@"C:\Development\Other\bedrock-server-1.14.1.4\worlds\BedrockGeneratedLevel\db"));
// db.Open();
// int x = 15;
// int z = 6;
// Log.Warn("Looking for version");
// var versionKey = BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x76}).ToArray();
// var version = db.Get(versionKey);
// Assert.AreEqual(15, version.First());
// Log.Warn("Looking for key");
// Assert.NotNull(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 0}).ToArray()));
// Assert.NotNull(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 1}).ToArray()));
// Assert.NotNull(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 2}).ToArray()));
// Assert.NotNull(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 3}).ToArray()));
// Assert.NotNull(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 4}).ToArray()));
// Assert.NotNull(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 5}).ToArray()));
// Assert.Null(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 6}).ToArray()));
// Assert.Null(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 7}).ToArray()));
// Assert.Null(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 8}).ToArray())); // Fail??
// Assert.Null(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 9}).ToArray()));
// Assert.Null(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 10}).ToArray()));
// Assert.Null(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 11}).ToArray()));
// Assert.Null(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 12}).ToArray()));
// Assert.Null(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 13}).ToArray()));
// Assert.Null(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 14}).ToArray()));
// Assert.Null(db.Get(BitConverter.GetBytes(x).Concat(BitConverter.GetBytes(z)).Concat(new byte[] {0x2f, 15}).ToArray()));
//}
private void ParseChunk(ReadOnlySpan <byte> data)
{
var reader = new SpanReader(data);
var version = reader.ReadByte();
Assert.AreEqual(8, version); // new palette-based chunk format
var storageSize = reader.ReadByte();
for (int i = 0; i < storageSize; i++)
{
var bitsPerBlock = reader.ReadByte() >> 1;
Assert.AreEqual(4, bitsPerBlock);
int numberOfBytes = 4096 / (32 / bitsPerBlock) * 4;
var blockData = reader.Read(numberOfBytes);
Assert.AreEqual(4096 / 2, blockData.Length);
int paletteSize = reader.ReadInt32();
Assert.AreEqual(12, paletteSize);
for (int j = 0; j < paletteSize; j++)
{
NbtFile file = new NbtFile();
file.BigEndian = false;
file.UseVarInt = false;
var buffer = data.Slice(reader.Position).ToArray();
int numberOfBytesRead = (int)file.LoadFromStream(new MemoryStream(buffer), NbtCompression.None);
reader.Position += numberOfBytesRead;
Console.WriteLine(file.RootTag);
Assert.NotZero(numberOfBytesRead);
}
}
}
internal ReadOnlySpan <byte> GetCurrentValue()
{
_reader.Position = (int)_lastValue.Offset;
return(_reader.Read(_lastValue.Length));
}
public void LevelDbWriteUserDataTest()
{
// Plan
var operations = new KeyValuePair <byte[], MemCache.ResultCacheEntry> [3];
for (int i = 0; i < 3; i++)
{
byte[] key = TestUtils.FillArrayWithRandomBytes(20);
var entry = new MemCache.ResultCacheEntry();
entry.ResultState = ResultState.Exist;
entry.Sequence = 10;
entry.Data = TestUtils.FillArrayWithRandomBytes(32768); // 32KB is maz size for a block, not that it matters for this
operations[i] = new KeyValuePair <byte[], MemCache.ResultCacheEntry>(key, entry);
}
MemCache memCache = new MemCache();
// Do
ReadOnlySpan <byte> result = memCache.EncodeBatch(operations);
// Check
SpanReader reader = new SpanReader(result);
Assert.AreEqual(10, reader.ReadInt64(), "Sequence number");
Assert.AreEqual(3, reader.ReadInt32(), "Operations count");
for (int i = 0; i < 3; i++)
{
var expectedKey = operations[i].Key;
var expectedData = operations[i].Value.Data;
Assert.AreEqual(1, reader.ReadByte(), "Operations type PUT");
var keyLen = reader.ReadVarLong();
Assert.AreEqual(expectedKey.Length, keyLen, "Key len");
Assert.AreEqual(expectedKey, reader.Read(keyLen).ToArray(), "Key");
var dataLen = reader.ReadVarLong();
Assert.AreEqual(expectedData.Length, dataLen, "Data len");
Assert.AreEqual(expectedData, reader.Read(dataLen).ToArray(), "Data");
}
// test encoding complete blocks
var stream = new MemoryStream();
LogWriter writer = new LogWriter(stream);
writer.WriteData(result);
Assert.Less(0, stream.Length);
stream.Position = 0;
// Roundtrip test by making sure i can read blocks I've encoded myself.
LogReader logReader = new LogReader(stream);
logReader.Open();
MemCache memCache2 = new MemCache();
memCache2.Load(logReader);
var cache = memCache2._resultCache;
Assert.AreEqual(3, cache.Count);
int j = 0;
foreach (var entry in cache)
{
var expectedKey = operations[j].Key;
var expectedData = operations[j].Value.Data;
Assert.AreEqual(ResultState.Exist, entry.Value.ResultState, "Value exists");
Assert.AreEqual(expectedKey.Length, entry.Key.Length, "Key len");
Assert.AreEqual(expectedKey, entry.Key, "Key");
Assert.AreEqual(expectedData.Length, entry.Value.Data.Length, "Data len");
Assert.AreEqual(expectedData, entry.Value.Data, "Data");
j++;
}
}
internal void EncodeBlocks(Stream stream, ReadOnlySpan <byte> data)
{
SpanReader reader = new SpanReader(data);
var currentRecordType = LogRecordType.Zero;
while (!reader.Eof)
{
int sizeLeft = (int)(BlockSize - stream.Position % BlockSize);
int bytesLeft = reader.Length - reader.Position;
int length = 0;
if (currentRecordType == LogRecordType.Zero || currentRecordType == LogRecordType.Last || currentRecordType == LogRecordType.Full)
{
if (sizeLeft < 7)
{
//throw new Exception($"Size left={sizeLeft}");
// pad with zeros
stream.Seek(sizeLeft, SeekOrigin.Current);
currentRecordType = LogRecordType.Zero;
continue;
}
if (sizeLeft == 7)
{
//throw new Exception($"Size left={sizeLeft}");
// emit empty first block
currentRecordType = LogRecordType.First;
WriteFragment(stream, currentRecordType, ReadOnlySpan <byte> .Empty);
continue;
}
if (sizeLeft >= bytesLeft + 7)
{
currentRecordType = LogRecordType.Full;
length = bytesLeft;
}
else
{
currentRecordType = LogRecordType.First;
length = sizeLeft - 7;
}
}
else if (currentRecordType == LogRecordType.First || currentRecordType == LogRecordType.Middle)
{
if (sizeLeft >= bytesLeft + 7)
{
currentRecordType = LogRecordType.Last;
length = bytesLeft;
}
else
{
currentRecordType = LogRecordType.Middle;
length = sizeLeft - 7;
}
}
else
{
throw new Exception("Unexpected state while writing fragments");
}
var fragmentData = reader.Read(length);
WriteFragment(stream, currentRecordType, fragmentData);
}
}
public void LevelDbReadLogTest()
{
// https://github.com/google/leveldb/blob/master/doc/log_format.md
DirectoryInfo directory = TestUtils.GetTestDirectory();
LogReader logReader = new LogReader(new FileInfo(Path.Combine(directory.FullName, "000047.log")));
BytewiseComparator comparator = new BytewiseComparator();
bool found = false;
while (true)
{
ReadOnlySpan <byte> data = logReader.ReadData();
if (logReader.Eof)
{
break;
}
var dataReader = new SpanReader(data);
long sequenceNumber = dataReader.ReadInt64();
long size = dataReader.ReadInt32();
while (!dataReader.Eof)
{
byte recType = dataReader.ReadByte();
ulong v1 = dataReader.ReadVarLong();
var currentKey = dataReader.Read(v1);
//CurrentKey = f5 ff ff ff eb ff ff ff 36
if (comparator.Compare(new byte[] { 0xf5, 0xff, 0xff, 0xff, 0xeb, 0xff, 0xff, 0xff, 0x36 }, currentKey) == 0)
{
Assert.False(found);
found = true;
}
ulong v2 = 0;
ReadOnlySpan <byte> currentVal = ReadOnlySpan <byte> .Empty;
switch (recType)
{
case 1: // value
{
if (recType == 1)
{
v2 = dataReader.ReadVarLong();
currentVal = dataReader.Read(v2);
}
break;
}
case 0: // delete
{
//Assert.Fail("Unexpected delete key");
break;
}
default:
throw new Exception("Unknown record format");
}
if (Log.IsDebugEnabled)
{
Log.Debug($"RecType={recType}, Sequence={sequenceNumber}, Size={size}, v1={v1}, v2={v2}\nCurrentKey={currentKey.HexDump(currentKey.Length, false, false)}\nCurrentVal=\n{currentVal.HexDump(cutAfterFive: true)} ");
}
}
}
Assert.True(found);
}
