public void Should_produce_a_well_formed_request_body_from_several_snapshots()
{
var data1 = Guid.NewGuid().ToByteArray();
var data2 = Guid.NewGuid().ToByteArray();
var data3 = Guid.NewGuid().ToByteArray();
var snapshot1 = Snapshot(data1, 2);
var snapshot2 = Snapshot(data2, 5);
var snapshot3 = Snapshot(data3, 1);
using (var content = factory.CreateContent(new[] { snapshot1, snapshot2, snapshot3 }, out var recordsCount, out var recordsSize))
{
var body = content.Value;
recordsCount.Should().Be(8);
recordsSize.Should().Be(48);
var reader = new BinaryBufferReader(body.Buffer, 0)
{
Endianness = Endianness.Big
};
reader.ReadInt32().Should().Be(recordsCount);
body.Buffer.Skip((int)reader.Position).Take((int)(body.Length - reader.Position))
.ToArray()
.Should()
.Equal(data1.Concat(data2).Concat(data3));
}
}
public BinaryBufferReaderTests()
{
_data = new byte[1024];
_mem = new MemoryStream(_data);
_writer = new BinaryWriter(_mem, Encoding.UTF8, true);
_bufferReader = new BinaryBufferReader(_data);
}
protected BinaryReaderVsBufferReaderBase()
{
_buffer = new byte[1024];
_mem = new MemoryStream(_buffer);
_binaryReader = new BinaryReader(_mem);
_bufferReader = new BinaryBufferReader(_buffer);
}
private static Uri ReadReplica([NotNull] BinaryBufferReader reader)
{
var host = reader.ReadString();
var port = reader.ReadInt32();
return(new Uri($"http://{host}:{port}"));
}
private static ReplicaWeight ReadWeight([NotNull] BinaryBufferReader reader)
{
var value = reader.ReadDouble();
var timestamp = reader.ReadDateTime();
return(new ReplicaWeight(value, timestamp));
}
public static CubeInfo[] ParseCubes(string cubeData, string colourData)
{
BinaryBufferReader cubes = new BinaryBufferReader(Convert.FromBase64String(cubeData), 0);
BinaryBufferReader colours = new BinaryBufferReader(Convert.FromBase64String(colourData), 0);
uint cubeCount = cubes.ReadUint();
uint colourCount = colours.ReadUint();
if (cubeCount != colourCount)
{
Logging.MetaLog("Something is f*****g broken");
return(null);
}
Logging.MetaLog($"Detected {cubeCount} cubes");
CubeInfo[] result = new CubeInfo[cubeCount];
for (int cube = 0; cube < cubeCount; cube++)
{
result[cube] = TranslateSpacialEnumerations(
cubes.ReadUint(),
cubes.ReadByte(),
cubes.ReadByte(),
cubes.ReadByte(),
cubes.ReadByte(),
colours.ReadByte(),
colours.ReadByte(),
colours.ReadByte(),
colours.ReadByte()
);
}
return(result);
}
public void TryAppend_should_correctly_assemble_message_from_multiple_record_groups()
{
builder.TryAppend("key1", CreateGroupBufferSlice("message1"));
builder.TryAppend("key2", CreateGroupBufferSlice("message22"));
builder.TryAppend("key3", CreateGroupBufferSlice("message333"));
builder.Message.Count.Should().Be(81);
var reader = new BinaryBufferReader(buffer, 0);
reader.ReadInt16().Should().Be(1); // version
reader.ReadInt32().Should().Be(3); // groups count
reader.ReadString().Should().Be("key1"); // routing key
reader.ReadInt32().Should().Be(1); // payload count
reader.ReadString().Should().Be("message1"); // payload message
reader.ReadString().Should().Be("key2"); // routing key
reader.ReadInt32().Should().Be(1); // payload count
reader.ReadString().Should().Be("message22"); // payload message
reader.ReadString().Should().Be("key3"); // routing key
reader.ReadInt32().Should().Be(1); // payload count
reader.ReadString().Should().Be("message333"); // payload message
}
private static ushort GetFieldsCount(byte[] buffer)
{
var reader = new BinaryBufferReader(buffer, 0)
{
Endianness = Endianness.Big
};
return(reader.ReadUInt16());
}
public void TrySerialize_should_correctly_serialize_record_given_sufficient_memory()
{
serializer.TrySerialize(item, itemSerializer, timestamp, buffer).Should().BeTrue();
var reader = new BinaryBufferReader(buffer.InternalBuffer, 0);
reader.ReadInt64().Should().Be(timestamp.ToUniversalTime().ToUnixTimeMilliseconds());
reader.ReadInt32().Should().Be(10);
reader.ReadString().Should().Be(item);
}
private static void Test <T>(T item, Action <T, IBinaryWriter> write, Func <IBinaryReader, T> read)
{
var writer = new BinaryBufferWriter(1);
write(item, writer);
var reader = new BinaryBufferReader(writer.Buffer, 0);
var readItem = read(reader);
readItem.ShouldBeEquivalentTo(item);
}
public static ApplicationInfo Deserialize([NotNull] string environment, [NotNull] string application, [CanBeNull] byte[] data)
{
if (data == null || data.Length == 0)
{
return(new ApplicationInfo(environment, application, null));
}
var reader = new BinaryBufferReader(data, 0);
var properties = reader.ReadDictionary(r => r.ReadString(), r => r.ReadString());
return(new ApplicationInfo(environment, application, properties));
}
private void TestSerialization()
{
var writer = new BinaryBufferWriter(64);
writer.Write(Guid.NewGuid());
serializer.Serialize(tree, writer);
var reader = new BinaryBufferReader(writer.Buffer, 16);
var deserializedTree = serializer.Deserialize(reader);
deserializedTree.Should().Be(tree);
}
private void TestNavigation(ClusterConfigPath path)
{
var writer = new BinaryBufferWriter(64);
writer.Write(Guid.NewGuid());
serializer.Serialize(tree, writer);
var reader = new BinaryBufferReader(writer.Buffer, 16);
var deserializedTree = serializer.Deserialize(reader, path.Segments);
deserializedTree.Should().Be(tree.ScopeTo(path.Segments));
}
private static void GetHeaderValues(ref BinaryBufferReader reader,
out int sectorSize,
out int shortSectorSize,
out int satSectorCount,
out int firstSecIdDirectoryStream,
out uint standardStreamSizeThreshold,
out int firstSecIdSsat,
out uint ssatSectorCount,
out int firstSecIdExtendedMsat,
out int msatExtraSectorCount)
{
if (!reader.ReadSpan(8).SequenceEqual(HEADER_SIGNATURE))
{
throw new CdfException(Errors.HeaderSignatureMissing);
}
reader.Position += 22;
var sectorSizeExponent = reader.ReadUInt16();
if (sectorSizeExponent < 7)
{
throw new CdfException(Errors.SectorSizeTooSmall);
}
sectorSize = (int)Math.Pow(2, sectorSizeExponent);
var shortSectorSizeExponent = reader.ReadUInt16();
if (shortSectorSizeExponent > sectorSizeExponent)
{
throw new CdfException(Errors.ShortSectorSizeGreaterThanStandardSectorSize);
}
shortSectorSize = (int)Math.Pow(2, shortSectorSizeExponent);
reader.Position += 10;
satSectorCount = (int)reader.ReadUInt32();
firstSecIdDirectoryStream = reader.ReadInt32();
reader.Position += 4;
standardStreamSizeThreshold = reader.ReadUInt32();
firstSecIdSsat = reader.ReadInt32();
ssatSectorCount = reader.ReadUInt32();
firstSecIdExtendedMsat = reader.ReadInt32();
msatExtraSectorCount = reader.ReadInt32();
}
public static ReplicaWeights Deserialize([NotNull] byte[] data)
{
var reader = new BinaryBufferReader(data, 0);
var count = reader.ReadInt32();
var weights = new ReplicaWeights(count);
for (var i = 0; i < count; i++)
{
var replica = ReadReplica(reader);
var weight = ReadWeight(reader);
weights[replica] = weight;
}
return(weights);
}
public static EnvironmentInfo Deserialize([NotNull] string environment, [CanBeNull] byte[] data)
{
if (data == null || data.Length == 0)
{
return(new EnvironmentInfo(environment, null, null));
}
var reader = new BinaryBufferReader(data, 0);
var version = reader.ReadInt32();
var parentEnvironment = reader.ReadNullable(r => r.ReadString());
var properties = version >= WithPropertiesVersion
? DeserializeProperties(reader)
: null;
return(new EnvironmentInfo(environment, parentEnvironment, properties));
}
public void Resize_performed_right_after_manual_position_change_should_not_lose_recent_data()
{
var writer = new BinaryBufferWriter(8);
writer.Write(1);
BitConverter.GetBytes(2).CopyTo(writer.Buffer, 4);
writer.Position += 4;
writer.EnsureCapacity(100);
writer.Position.Should().Be(8);
writer.Length.Should().Be(8);
var reader = new BinaryBufferReader(writer.Buffer, 0);
reader.ReadInt32().Should().Be(1);
reader.ReadInt32().Should().Be(2);
}
// (iloktionov): Работает в предположении о том, что размер одной записи в буфере не может превышать maximumSliceLength.
public IEnumerable <BufferSlice> Cut(IBuffer buffer, int maximumSliceLength)
{
if (buffer.SnapshotLength <= maximumSliceLength)
{
yield return(new BufferSlice(buffer, 0, buffer.SnapshotLength, buffer.SnapshotCount));
yield break;
}
var currentOffset = 0;
var currentSize = 0;
var currentCount = 0;
var reader = new BinaryBufferReader(buffer.InternalBuffer, 0);
for (var i = 0; i < buffer.SnapshotCount; i++)
{
var recordLength = ReadRecordLength(reader);
if (recordLength > maximumSliceLength)
{
throw new InvalidOperationException($"Bug! Encountered a record with length = {recordLength} greater than max slice size {maximumSliceLength}.");
}
if (currentSize + recordLength > maximumSliceLength)
{
yield return(new BufferSlice(buffer, currentOffset, currentSize, currentCount));
currentOffset += currentSize;
currentSize = 0;
currentCount = 0;
}
currentSize += recordLength;
currentCount++;
}
if (currentSize > 0)
{
yield return(new BufferSlice(buffer, currentOffset, currentSize, currentCount));
}
}
private static int[] BuildSatSecIdChain(BinaryBufferReader reader, int msatExtraSectorCount, int firstSecIdExtendedMsat, int satSectorCount, int sectorSize, int secIdsPerSector)
{
#if DEBUG
var msat = new int[1 + msatExtraSectorCount + 1];
var sat = new int[satSectorCount + 1];
msat[0] = SECID_MSAT;
#else
var sat = new int[satSectorCount];
#endif
var firstPartMsatSatSectorCount = Math.Min(satSectorCount, HEADER_MSAT_SAT_SECID_COUNT);
var remainder = firstPartMsatSatSectorCount % 4;
for (var i = 0; i < remainder; i++)
{
sat[i] = reader.ReadInt32();
}
if (firstPartMsatSatSectorCount >= 4)
{
var remainingSecIdCount = firstPartMsatSatSectorCount - remainder;
for (var i = 0; i < remainingSecIdCount; i += 4)
{
sat[remainder + i] = reader.ReadInt32();
sat[remainder + i + 1] = reader.ReadInt32();
sat[remainder + i + 2] = reader.ReadInt32();
sat[remainder + i + 3] = reader.ReadInt32();
}
}
if (firstPartMsatSatSectorCount < satSectorCount)
{
var satSectorIndex = (int)HEADER_MSAT_SAT_SECID_COUNT;
var remainingMsatSatSectorCount = satSectorCount - HEADER_MSAT_SAT_SECID_COUNT;
var currentSecIdMsat = firstSecIdExtendedMsat;
var currentSectorPosMsat = HEADER_SIZE + currentSecIdMsat * sectorSize;
for (var i = 0; i < msatExtraSectorCount; i++)
{
#if DEBUG
msat[i + 1] = currentSecIdMsat;
#endif
reader.Position = currentSectorPosMsat;
var remainingSecIdsInCurrentSector = Math.Min(remainingMsatSatSectorCount, secIdsPerSector - 1);
remainder = remainingSecIdsInCurrentSector % 4;
for (var j = 0; j < remainder; j++)
{
sat[satSectorIndex++] = reader.ReadInt32();
remainingMsatSatSectorCount--;
}
if (remainingSecIdsInCurrentSector >= 4)
{
var remainingSecIdCount = remainingSecIdsInCurrentSector - remainder;
for (var j = 0; j < remainingSecIdCount; j += 4)
{
sat[satSectorIndex] = reader.ReadInt32();
sat[satSectorIndex + 1] = reader.ReadInt32();
sat[satSectorIndex + 2] = reader.ReadInt32();
sat[satSectorIndex + 3] = reader.ReadInt32();
remainingMsatSatSectorCount -= 4;
satSectorIndex += 4;
}
}
if (remainingMsatSatSectorCount > 0)
{
currentSecIdMsat = reader.ReadInt32();
currentSectorPosMsat = HEADER_SIZE + currentSecIdMsat * sectorSize;
}
#if DEBUG
else
{
msat[1 + i + 1] = SECID_END_OF_CHAIN;
sat[satSectorIndex] = SECID_END_OF_CHAIN;
}
#endif
}
}
#if DEBUG
else
{
msat[^ 1] = SECID_END_OF_CHAIN;
public void Initialize()
{
m_Writer = new BinaryBufferWriter(64);
m_Reader = new BinaryBufferReader(m_Writer.Buffer);
}
private static Dictionary <string, string> DeserializeProperties(BinaryBufferReader reader)
{
return(reader.ReadDictionary(r => r.ReadString(), r => r.ReadString()));
}
public ByteBufferAirlockSource(byte[] buffer)
{
Reader = new BinaryBufferReader(buffer, 0);
}
internal Dictionary <string, byte[]> Mount(byte[] data, Predicate <string> streamNameMatch, bool?returnOnFirstMatch, bool rootStorageDescendantsOnly)
{
var mainReader = new BinaryBufferReader(data);
try
{
GetHeaderValues(ref mainReader,
out var sectorSize,
out var shortSectorSize,
out var satSectorCount,
out var firstSecIdDirectoryStream,
out var standardStreamSizeThreshold,
out var firstSecIdSsat,
out var ssatSectorCount,
out var firstSecIdExtendedMsat,
out var msatExtraSectorCount);
// Master Sector Allocation Table (MSAT) / Sector Allocation Table (SAT)
var secIdsPerSector = sectorSize / SECID_SIZE;
var satSecIdChain = BuildSatSecIdChain(mainReader, msatExtraSectorCount, firstSecIdExtendedMsat, satSectorCount, sectorSize, secIdsPerSector);
// Short-Sector Allocation Table (SSAT)
var ssatSecIdChain = BuildSsatSecIdChain(mainReader, ssatSectorCount, satSecIdChain, firstSecIdSsat, sectorSize, secIdsPerSector);
// Directory
var directoryStream = ReadDirectoryStream(mainReader, firstSecIdDirectoryStream, satSecIdChain, sectorSize);
DirectoryEntries = new List <DirectoryEntry>(directoryStream.Length / DIRECTORY_ENTRY_SIZE);
if (rootStorageDescendantsOnly)
{
ReadDirectoryEntries(mainReader, null, null, directoryStream, satSecIdChain, sectorSize, standardStreamSizeThreshold, shortSectorSize, ssatSecIdChain);
VisitEntries();
if (streamNameMatch != null)
{
if (returnOnFirstMatch.HasValue && returnOnFirstMatch.Value)
{
var matchedDirectoryEntry = DirectoryEntries.FirstOrDefault(de => streamNameMatch(de.Name));
return(matchedDirectoryEntry != null ? new Dictionary <string, byte[]> {
[matchedDirectoryEntry.Name] = matchedDirectoryEntry.Stream
} : new Dictionary <string, byte[]>());
}
return(DirectoryEntries.Where(de => streamNameMatch(de.Name)).ToDictionary(de => de.Name, de => de.Stream));
}
return(null);
}
return(ReadDirectoryEntries(mainReader, streamNameMatch, returnOnFirstMatch, directoryStream, satSecIdChain, sectorSize, standardStreamSizeThreshold, shortSectorSize, ssatSecIdChain));
}
catch (EndOfStreamException e)
{
throw new CdfException(Errors.UnexpectedEndOfStream, e);
}
}
