public void OnEncoding(Token fieldToken, DirectBuffer buffer, int index, Token typeToken, int actingVersion)
{
string value = ReadEncodingAsString(buffer, index, typeToken, actingVersion);
PrintScope();
Console.WriteLine("{0}={1}", fieldToken.Name, value);
}
public static int Decode(MessageHeader messageHeader,
MarketDataIncrementalRefreshTrades marketData,
DirectBuffer buffer,
int bufferIndex)
{
messageHeader.Wrap(buffer, bufferIndex, 0);
int actingVersion = messageHeader.Version;
int actingBlockLength = messageHeader.BlockLength;
marketData.WrapForDecode(buffer, bufferIndex + MessageHeader.Size, actingBlockLength, actingVersion);
var transactTime = marketData.TransactTime;
var matchEventIndicator = marketData.MatchEventIndicator;
var mdIncGrpGroup = marketData.MdIncGrp;
while (mdIncGrpGroup.HasNext)
{
mdIncGrpGroup.Next();
var tradeId = mdIncGrpGroup.TradeId;
var securityId = mdIncGrpGroup.SecurityId;
var mantissa = mdIncGrpGroup.MdEntryPx.Mantissa;
var i = mdIncGrpGroup.MdEntrySize.Mantissa;
var numberOfOrders = mdIncGrpGroup.NumberOfOrders;
var mdUpdateAction = mdIncGrpGroup.MdUpdateAction;
var rptSeq = mdIncGrpGroup.RptSeq;
var aggressorSide = mdIncGrpGroup.AggressorSide;
var mdEntryType = mdIncGrpGroup.MdEntryType;
}
return marketData.Size;
}
public IrDecoder(byte[] buffer)
{
_size = buffer.Length;
_directBuffer = new DirectBuffer(buffer);
_offset = 0;
_valBuffer = new DirectBuffer(_valArray);
}
private static int DecodeField(DirectBuffer buffer, int bufferIndex, IList<Token> tokens, int fromIndex,
int actingVersion, ITokenListener listener)
{
int toIndex = FindNextOrLimit(tokens, fromIndex + 1, tokens.Count, Signal.EndField);
Token fieldToken = tokens[fromIndex];
Token typeToken = tokens[fromIndex + 1];
switch (typeToken.Signal)
{
case Signal.BeginComposite:
DecodeComposite(fieldToken, buffer, bufferIndex + typeToken.Offset, tokens, fromIndex + 1,
toIndex - 1, actingVersion, listener);
break;
case Signal.BeginEnum:
listener.OnEnum(fieldToken, buffer, bufferIndex + typeToken.Offset, tokens, fromIndex + 1,
toIndex - 1, actingVersion);
break;
case Signal.BeginSet:
listener.OnBitSet(fieldToken, buffer, bufferIndex + typeToken.Offset, tokens, fromIndex + 1,
toIndex - 1, actingVersion);
break;
case Signal.Encoding:
listener.OnEncoding(fieldToken, buffer, bufferIndex + typeToken.Offset, typeToken, actingVersion);
break;
}
return toIndex;
}
public IrEncoder(string fileName, IntermediateRepresentation ir)
{
Channel = (new FileOutputStream(fileName)).Channel;
ResultBuffer = null;
Buffer = ByteBuffer.allocateDirect(CAPACITY);
DirectBuffer = new DirectBuffer(Buffer);
this.Ir = ir;
}
public IrEncoder(ByteBuffer buffer, IntermediateRepresentation ir)
{
Channel = null;
ResultBuffer = buffer;
this.Buffer = ByteBuffer.allocateDirect(CAPACITY);
DirectBuffer = new DirectBuffer(this.Buffer);
this.Ir = ir;
}
public void SetUp()
{
_buffer = new Byte[16];
_directBuffer = new DirectBuffer(_buffer);
var handle = GCHandle.Alloc(_buffer, GCHandleType.Pinned);
_pBuffer = (byte*)handle.AddrOfPinnedObject().ToPointer();
}
public IrDecoder(string fileName)
{
byte[] buffer = File.ReadAllBytes(fileName);
_directBuffer = new DirectBuffer(buffer);
_size = buffer.Length;
_offset = 0;
_valBuffer = new DirectBuffer(_valArray);
}
public static PrimitiveValue Get(DirectBuffer buffer, PrimitiveType type, int length)
{
if (length == 0)
{
return null;
}
switch (type.Type)
{
case SbePrimitiveType.Char:
if (length == 1)
{
return new PrimitiveValue(buffer.CharGet(0), 1);
}
else
{
var array = new byte[length];
buffer.GetBytes(0, array, 0, array.Length);
return new PrimitiveValue(array, "UTF-8", array.Length);
}
case SbePrimitiveType.Int8:
return new PrimitiveValue(buffer.Int8Get(0), 1);
case SbePrimitiveType.Int16:
return new PrimitiveValue(buffer.Int16GetLittleEndian(0), 2);
case SbePrimitiveType.Int32:
return new PrimitiveValue(buffer.Int32GetLittleEndian(0), 4);
case SbePrimitiveType.Int64:
return new PrimitiveValue(buffer.Int64GetLittleEndian(0), 8);
case SbePrimitiveType.UInt8:
return new PrimitiveValue(buffer.Uint8Get(0), 1);
case SbePrimitiveType.UInt16:
return new PrimitiveValue(buffer.Uint16GetLittleEndian(0), 2);
case SbePrimitiveType.UInt32:
return new PrimitiveValue(buffer.Uint32GetLittleEndian(0), 4);
case SbePrimitiveType.UInt64:
return new PrimitiveValue(buffer.Uint64GetLittleEndian(0), 8);
case SbePrimitiveType.Float:
return new PrimitiveValue(buffer.FloatGetLittleEndian(0), 4);
case SbePrimitiveType.Double:
return new PrimitiveValue(buffer.DoubleGetLittleEndian(0), 8);
default:
return null;
}
}
private static void DecodeFields(DirectBuffer buffer, int bufferIndex, int actingVersion, IList<Token> tokens,
int fromIndex, int toIndex, ITokenListener listener)
{
for (int i = fromIndex; i < toIndex; i++)
{
if (Signal.BeginField == tokens[i].Signal)
{
i = DecodeField(buffer, bufferIndex, tokens, i, actingVersion, listener);
}
}
}
private static void DecodeComposite(Token fieldToken, DirectBuffer buffer, int bufferIndex, IList<Token> tokens,
int fromIndex, int toIndex, int actingVersion, ITokenListener listener)
{
listener.OnBeginComposite(fieldToken, tokens, fromIndex, toIndex);
for (int i = fromIndex + 1; i < toIndex; i++)
{
Token token = tokens[i];
listener.OnEncoding(token, buffer, bufferIndex + token.Offset, token, actingVersion);
}
listener.OnEndComposite(fieldToken, tokens, fromIndex, toIndex);
}
public void ConstructFromNativeBuffer()
{
var managedBuffer = new Byte[16];
var handle = GCHandle.Alloc(managedBuffer, GCHandleType.Pinned);
var unmanagedBuffer = (byte*) handle.AddrOfPinnedObject().ToPointer();
const int value = 5;
const int index = 0;
using (var directBufferFromUnmanagedbuffer = new DirectBuffer(unmanagedBuffer, managedBuffer.Length))
{
directBufferFromUnmanagedbuffer.Int64PutLittleEndian(index, value);
Assert.AreEqual(value, *(long*) (unmanagedBuffer + index));
}
}
private static int DecodeGroups(DirectBuffer buffer, int bufferIndex, int actingVersion, IList<Token> tokens,
int fromIndex, int toIndex, ITokenListener listener)
{
for (int i = fromIndex; i < toIndex; i++)
{
Token token = tokens[i];
if (Signal.BeginGroup == token.Signal)
{
Token blockLengthToken = tokens[i + 2];
int blockLength = Util.GetInt(buffer, bufferIndex + blockLengthToken.Offset,
blockLengthToken.Encoding.PrimitiveType, blockLengthToken.Encoding.ByteOrder);
Token numInGroupToken = tokens[i + 3];
int numInGroup = Util.GetInt(buffer, bufferIndex + numInGroupToken.Offset,
numInGroupToken.Encoding.PrimitiveType, numInGroupToken.Encoding.ByteOrder);
Token dimensionTypeComposite = tokens[i + 1];
bufferIndex += dimensionTypeComposite.Size;
int beginFieldsIndex = i + GroupDimTypeTokens;
int endGroupIndex = FindNextOrLimit(tokens, beginFieldsIndex, toIndex, Signal.EndGroup);
int nextGroupIndex = FindNextOrLimit(tokens, beginFieldsIndex, toIndex, Signal.BeginGroup);
int endOfFieldsIndex = Math.Min(endGroupIndex, nextGroupIndex) - 1;
for (int g = 0; g < numInGroup; g++)
{
listener.OnBeginGroup(token, g, numInGroup);
DecodeFields(buffer, bufferIndex, actingVersion, tokens, beginFieldsIndex, endOfFieldsIndex,
listener);
bufferIndex += blockLength;
if (nextGroupIndex < endGroupIndex)
{
bufferIndex = DecodeGroups(buffer, bufferIndex, actingVersion, tokens, nextGroupIndex,
toIndex, listener);
}
listener.OnEndGroup(token, g, numInGroup);
}
i = endGroupIndex;
}
}
return bufferIndex;
}
private static void EncodeTestMessage(byte[] buffer)
{
var directBuffer = new DirectBuffer(buffer);
int bufferOffset = 0;
MESSAGE_HEADER.Wrap(directBuffer, bufferOffset, ActingVersion);
MESSAGE_HEADER.BlockLength = Car.BlockLength;
MESSAGE_HEADER.TemplateId = Car.TemplateId;
MESSAGE_HEADER.Version = Car.TemplateVersion;
bufferOffset += MessageHeader.Size;
bufferOffset += ExampleUsingGeneratedStub.Encode(Car, directBuffer, bufferOffset);
//buffer.po (bufferOffset);
}
public void Recycle()
{
var directBuffer = new DirectBuffer();
var firstBuffer = new Byte[16];
directBuffer.Wrap(firstBuffer);
directBuffer.Int64PutLittleEndian(0, 1);
Assert.AreEqual(1, BitConverter.ToInt64(firstBuffer, 0));
var secondBuffer = new byte[16];
var secondBufferHandle = GCHandle.Alloc(secondBuffer, GCHandleType.Pinned);
var secondUnmanagedBuffer = (byte*)secondBufferHandle.AddrOfPinnedObject().ToPointer();
directBuffer.Wrap(secondUnmanagedBuffer, 16);
directBuffer.Int64PutLittleEndian(0, 2);
Assert.AreEqual(2, BitConverter.ToInt64(secondBuffer, 0));
directBuffer.Dispose();
}
public static int Encode(MessageHeader messageHeader,
MarketDataIncrementalRefreshTrades marketData,
DirectBuffer buffer,
int bufferIndex)
{
messageHeader.Wrap(buffer, bufferIndex, 0);
messageHeader.BlockLength = MarketDataIncrementalRefreshTrades.BlockLength;
messageHeader.TemplateId = MarketDataIncrementalRefreshTrades.TemplateId;
messageHeader.SchemaId = MarketDataIncrementalRefreshTrades.SchemaId;
messageHeader.Version = MarketDataIncrementalRefreshTrades.Schema_Version;
marketData.WrapForEncode(buffer, bufferIndex + MessageHeader.Size);
marketData.TransactTime = 1234L;
marketData.EventTimeDelta = 987;
marketData.MatchEventIndicator = MatchEventIndicator.END_EVENT;
var mdIncGrp = marketData.MdIncGrpCount(2);
mdIncGrp.Next();
mdIncGrp.TradeId = 1234L;
mdIncGrp.SecurityId = 56789L;
mdIncGrp.MdEntryPx.Mantissa = 50;
mdIncGrp.MdEntrySize.Mantissa = 10;
mdIncGrp.NumberOfOrders = 1;
mdIncGrp.MdUpdateAction = MDUpdateAction.NEW;
mdIncGrp.RptSeq = 1;
mdIncGrp.AggressorSide = Side.BUY;
mdIncGrp.MdEntryType = MDEntryType.BID;
mdIncGrp.Next();
mdIncGrp.TradeId = 1234L;
mdIncGrp.SecurityId = 56789L;
mdIncGrp.MdEntryPx.Mantissa = 50;
mdIncGrp.MdEntrySize.Mantissa = 10;
mdIncGrp.NumberOfOrders = 1;
mdIncGrp.MdUpdateAction = MDUpdateAction.NEW;
mdIncGrp.RptSeq = 1;
mdIncGrp.AggressorSide = Side.SELL;
mdIncGrp.MdEntryType = MDEntryType.OFFER;
return marketData.Size;
}
public static unsafe bool TryGetDuplicate(this ReadOnlyTransaction tx, Database db, Bufferable key, ref Bufferable value)
{
using (var c = db.OpenReadOnlyCursor(tx))
{
fixed(byte *keyPtr = &key.Buffer[0], valPtr = &value.Buffer[0])
{
var keydb = new DirectBuffer(key.Buffer.Length, keyPtr);
var valuedb = new DirectBuffer(value.Buffer.Length, valPtr);
var success = c.TryFindDup(Lookup.EQ, ref keydb, ref valuedb);
if (success)
{
value = new Bufferable(valuedb.Span.ToArray());
}
return(success);
}
}
}
public void Recycle()
{
var directBuffer = new DirectBuffer();
var firstBuffer = new Byte[16];
directBuffer.Wrap(firstBuffer);
directBuffer.Int64PutLittleEndian(0, 1);
Assert.AreEqual(1, BitConverter.ToInt64(firstBuffer, 0));
var secondBuffer = new byte[16];
var secondBufferHandle = GCHandle.Alloc(secondBuffer, GCHandleType.Pinned);
var secondUnmanagedBuffer = (byte *)secondBufferHandle.AddrOfPinnedObject().ToPointer();
directBuffer.Wrap(secondUnmanagedBuffer, 16);
directBuffer.Int64PutLittleEndian(0, 2);
Assert.AreEqual(2, BitConverter.ToInt64(secondBuffer, 0));
directBuffer.Dispose();
}
/// <summary>
/// Iterate over dupsorted values by key.
/// </summary>
public IEnumerable <TValue> AsEnumerable <TKey, TValue>(ReadOnlyTransaction txn, TKey key)
{
var keyPtr = AsPointer(ref key);
var keyLength = TypeHelper <TKey> .EnsureFixedSize();
var key1 = new DirectBuffer(keyLength, (byte *)keyPtr);
var fixedMemory = BufferPool.Retain(keyLength, true);
key1.Span.CopyTo(fixedMemory.Span);
return(AsEnumerable(txn, fixedMemory).Select(buf =>
{
var valueSize = TypeHelper <TValue> .EnsureFixedSize();
if (buf.Length != valueSize)
{
throw new InvalidOperationException("Buffer length does not equals to value size");
}
return buf.Read <TValue>(0);
}));
}
public void OnEnum(Token fieldToken, DirectBuffer buffer, int bufferIndex, IList<Token> tokens, int beginIndex,
int endIndex, int actingVersion)
{
Token typeToken = tokens[beginIndex + 1];
long encodedValue = ReadEncodingAsLong(buffer, bufferIndex, typeToken, actingVersion);
string value = null;
for (int i = beginIndex + 1; i < endIndex; i++)
{
// TODO to check..
if (encodedValue == tokens[i].Encoding.ConstValue.LongValue())
{
value = tokens[i].Name;
break;
}
}
PrintScope();
Console.WriteLine("{0}={1}", fieldToken.Name, value);
}
public void Update(DirectBuffer input)
{
if (f[0] != 0)
{
ThrowHelper.HashFinalized();
}
var s = (Blake2bContext *)Unsafe.AsPointer(ref Unsafe.AsRef(this));
uint consumed = 0;
uint remaining = (uint)input.Length;
uint blockrem = BlockBytes - c;
if ((c != 0) && (remaining > blockrem))
{
if (blockrem != 0)
{
Unsafe.CopyBlockUnaligned(ref b[c], ref *input.Data, blockrem);
}
c = 0;
compress(s->b, 0, BlockBytes);
consumed += blockrem;
remaining -= blockrem;
}
if (remaining > BlockBytes)
{
uint cb = (remaining - 1) & ~((uint)BlockBytes - 1);
compress(input.Data, consumed, cb);
consumed += cb;
remaining -= cb;
}
if (remaining != 0)
{
Unsafe.CopyBlockUnaligned(ref b[c], ref *(input.Data + consumed), remaining);
c += remaining;
}
}
public LogBuffers(string logFileName, int termLength = LogBufferDescriptor.TERM_MIN_LENGTH)
{
try {
long logLength = LogBufferDescriptor.PARTITION_COUNT *
(LogBufferDescriptor.TERM_META_DATA_LENGTH + termLength) +
LogBufferDescriptor.LOG_META_DATA_LENGTH;
termLength = LogBufferDescriptor.ComputeTermLength(logLength);
LogBufferDescriptor.CheckTermLength(termLength);
_df = new DirectFile(logFileName, logLength);
_termLength = termLength;
// if log length exceeds MAX_INT we need multiple mapped buffers, (see FileChannel.map doc).
if (logLength < int.MaxValue)
{
int metaDataSectionOffset = termLength * LogBufferDescriptor.PARTITION_COUNT;
for (int i = 0; i < LogBufferDescriptor.PARTITION_COUNT; i++)
{
int metaDataOffset = metaDataSectionOffset + (i * LogBufferDescriptor.TERM_META_DATA_LENGTH);
_buffers[i] = new DirectBuffer(termLength, _df.Buffer.Data + i * termLength);
_buffers[i + LogBufferDescriptor.PARTITION_COUNT] = new DirectBuffer(LogBufferDescriptor.TERM_META_DATA_LENGTH, _df.Buffer.Data + metaDataOffset);
_partitions[i] = new LogBufferPartition(_buffers[i], _buffers[i + LogBufferDescriptor.PARTITION_COUNT]);
}
_buffers[_buffers.Length - 1] = new DirectBuffer(LogBufferDescriptor.LOG_META_DATA_LENGTH,
_df.Buffer.Data + (int)(logLength - LogBufferDescriptor.LOG_META_DATA_LENGTH));
}
else
{
throw new NotImplementedException("TODO Check .NET mapping limit");
}
} catch (IOException ex) {
throw new AggregateException(ex);
}
foreach (var buffer in _buffers)
{
buffer.VerifyAlignment(8);
}
}
public String DecodeSBEMessageQATEngine(byte[] MessageBytes)
{
StringBuilder sb = new StringBuilder();
var directBuffer = new DirectBuffer(MessageBytes);
int bufferOffset = 0;
ushort blockLength = directBuffer.Uint16GetLittleEndian(bufferOffset);
bufferOffset += 2;
sb.AppendLine(CrackHelper.CreateLine("Header", "BlockLength", blockLength.ToString()));
ushort templateId = directBuffer.Uint16GetLittleEndian(bufferOffset);
bufferOffset += 2;
sb.AppendLine(CrackHelper.CreateLine("Header", "TemplateId", templateId.ToString()));
ushort schemaId = directBuffer.Uint16GetLittleEndian(bufferOffset);
bufferOffset += 2;
sb.AppendLine(CrackHelper.CreateLine("Header", "SchemaId", schemaId.ToString()));
ushort version = directBuffer.Uint16GetLittleEndian(bufferOffset);
bufferOffset += 2;
sb.AppendLine(CrackHelper.CreateLine("Header", "Version", version.ToString()));
SbeMessage Message = SbeLoader.LoadMessageById(templateId.ToString());
if (Message == null)
{
throw new Exception("Message not recognized! TemplateId = " + templateId.ToString());
}
foreach (SbeField field in Message.Fields)
{
FillValue(directBuffer, field, ref bufferOffset);
}
sb.Append(Message.Crack());
return(sb.ToString());
}
public static void ExampleMain()
{
// This byte array is used for encoding and decoding, this is what you would send on the wire or save to disk
var byteBuffer = new byte[4096];
// You need to "wrap" the array with a DirectBuffer, this class is used by the generated code to read and write efficiently to the underlying byte array
var directBuffer = new DirectBuffer(byteBuffer);
const short SchemaVersion = 0;
int bufferOffset = 0;
var MessageHeader = new MessageHeader();
var Car = new Car();
// Before encoding a message we need to create a SBE header which specify what we are going to encode (this will allow the decoder to detect that it's an encoded 'car' object)
// We will probably simplify this part soon, so the header gets applied automatically, but for now it's manual
MessageHeader.Wrap(directBuffer, bufferOffset, Car.SchemaVersion); // position the MessageHeader on the DirectBuffer, at the correct position
MessageHeader.BlockLength = Car.BlockLength; // size that a car takes on the wire
MessageHeader.SchemaId = Car.SchemaId;
MessageHeader.TemplateId = Car.TemplateId; // identifier for the car object (SBE template ID)
MessageHeader.Version = Car.SchemaVersion; // this can be overriden if we want to support different versions of the car object (advanced functionality)
// Now that we have encoded the header in the byte array we can encode the car object itself
bufferOffset += MessageHeader.Size;
CarExample.Encode(Car, directBuffer, bufferOffset);
// Now we have encoded the message is the byte array, we are going to decode it
// first we decode the header (in a real world scenario you would need the header to decide which SBE decoder you are going to use
bufferOffset = 0;
// position the MessageHeader object at the beginning of the array
MessageHeader.Wrap(directBuffer, bufferOffset, SchemaVersion);
// Extract info from the header
// In a real app you would use that to lookup the applicable flyweight to decode this type of message based on templateId and version.
int actingBlockLength = MessageHeader.BlockLength;
int actingVersion = MessageHeader.Version;
bufferOffset += MessageHeader.Size;
// now we decode the message
CarExample.Decode(Car, directBuffer, bufferOffset, actingBlockLength, actingVersion);
}
public static int Encode(MessageHeader messageHeader,
MarketDataIncrementalRefreshTrades marketData,
DirectBuffer buffer,
int bufferIndex)
{
messageHeader.Wrap(buffer, bufferIndex, 0);
messageHeader.TemplateId = MarketDataIncrementalRefreshTrades.TemplateId;
messageHeader.Version = MarketDataIncrementalRefreshTrades.TemplateVersion;
messageHeader.BlockLength = MarketDataIncrementalRefreshTrades.BlockLength;
marketData.WrapForEncode(buffer, bufferIndex + MessageHeader.Size);
marketData.TransactTime = 1234L;
marketData.EventTimeDelta = 987;
marketData.MatchEventIndicator = MatchEventIndicator.END_EVENT;
var mdIncGrp = marketData.MdIncGrpCount(2);
mdIncGrp.Next();
mdIncGrp.TradeId = 1234L;
mdIncGrp.SecurityId = 56789L;
mdIncGrp.MdEntryPx.Mantissa = 50;
mdIncGrp.MdEntrySize.Mantissa = 10;
mdIncGrp.NumberOfOrders = 1;
mdIncGrp.MdUpdateAction = MDUpdateAction.NEW;
mdIncGrp.RptSeq = 1;
mdIncGrp.AggressorSide = Side.BUY;
mdIncGrp.MdEntryType = MDEntryType.BID;
mdIncGrp.Next();
mdIncGrp.TradeId = 1234L;
mdIncGrp.SecurityId = 56789L;
mdIncGrp.MdEntryPx.Mantissa = 50;
mdIncGrp.MdEntrySize.Mantissa = 10;
mdIncGrp.NumberOfOrders = 1;
mdIncGrp.MdUpdateAction = MDUpdateAction.NEW;
mdIncGrp.RptSeq = 1;
mdIncGrp.AggressorSide = Side.SELL;
mdIncGrp.MdEntryType = MDEntryType.OFFER;
return(marketData.Size);
}
/// <summary>
/// Decode a message from the provided buffer based on the message schema described with IR
/// <seealso cref="Token" />s.
/// </summary>
/// <param name="buffer"> containing the encoded message. </param>
/// <param name="bufferIndex"> at which the message encoding starts in the buffer. </param>
/// <param name="actingVersion"> of the encoded message for dealing with extension fields. </param>
/// <param name="blockLength"> of the root message fields. </param>
/// <param name="msgTokens"> in IR format describing the message structure. </param>
/// <param name="listener"> to callback for decoding the primitive values as discovered in the structure. </param>
/// <returns> the index in the underlying buffer after decoding. </returns>
public static int Decode(DirectBuffer buffer, int bufferIndex, int actingVersion, int blockLength,
IList<Token> msgTokens, ITokenListener listener)
{
int groupsBeginIndex = FindNextOrLimit(msgTokens, 1, msgTokens.Count, Signal.BeginGroup);
int varDataSearchStart = groupsBeginIndex != msgTokens.Count ? groupsBeginIndex : 1;
int varDataBeginIndex = FindNextOrLimit(msgTokens, varDataSearchStart, msgTokens.Count, Signal.BeginVarData);
listener.OnBeginMessage(msgTokens[0]);
DecodeFields(buffer, bufferIndex, actingVersion, msgTokens, 0, groupsBeginIndex, listener);
bufferIndex += blockLength;
bufferIndex = DecodeGroups(buffer, bufferIndex, actingVersion, msgTokens, groupsBeginIndex,
varDataBeginIndex, listener);
bufferIndex = DecodeVarData(buffer, bufferIndex, msgTokens, varDataBeginIndex, msgTokens.Count, listener);
listener.OnEndMessage(msgTokens[msgTokens.Count - 1]);
return bufferIndex;
}
public void RemoveSubFingerprint(SubFingerprintDataDTO subFingerprintDataDTO)
{
var subFingerprintKey = BitConverter.GetBytes(subFingerprintDataDTO.SubFingerprintReference).AsMemory();
using (subFingerprintKey.Pin())
{
var keyBuffer = new DirectBuffer(subFingerprintKey.Span);
databasesHolder.SubFingerprintsDatabase.Delete(tx, ref keyBuffer);
var trackKey = BitConverter.GetBytes(subFingerprintDataDTO.TrackReference).AsMemory();
using (trackKey.Pin())
{
var trackKeyBuffer = new DirectBuffer(trackKey.Span);
using var cursor = indexesHolder.TracksSubfingerprintsIndex.OpenCursor(tx);
if (cursor.TryGet(ref trackKeyBuffer, ref keyBuffer, CursorGetOption.GetBoth))
{
cursor.Delete(false);
}
}
}
}
public void CouldSetTypeEnumInJson()
{
#pragma warning disable 618
Settings.DoAdditionalCorrectnessChecks = false;
#pragma warning restore 618
var rm = BufferPool.Retain(10_000);
var db = new DirectBuffer(rm.Span);
var value = new SampleStruct(42);
var size = BinarySerializer.SizeOf(in value, out var tmpBuffer, SerializationFormat.Json);
var written = BinarySerializer.Write(in value, ref db, tmpBuffer, SerializationFormat.Json);
Assert.AreEqual(size, written);
var typeEnumByte = db[1];
Assert.AreEqual(123, typeEnumByte);
rm.Dispose();
}
public TrackDataDTO GetTrackById(string id)
{
var idKey = Encoding.UTF8.GetBytes(id).AsMemory();
using (idKey.Pin())
{
var keyBuffer = new DirectBuffer(idKey.Span);
if (indexesHolder.IdIndex.TryGet(tx, ref keyBuffer, out var valueBuffer))
{
var trackId = valueBuffer.ReadUInt64(0);
var trackKey = BitConverter.GetBytes(trackId).AsMemory();
using (trackKey.Pin())
{
var trackKeyBuffer = new DirectBuffer(trackKey.Span);
return(GetTrackById(ref trackKeyBuffer, tx));
}
}
return(null);
}
}
/// <summary>
/// Decode a message from the provided buffer based on the message schema described with IR
/// <seealso cref="Token" />s.
/// </summary>
/// <param name="buffer"> containing the encoded message. </param>
/// <param name="bufferIndex"> at which the message encoding starts in the buffer. </param>
/// <param name="actingVersion"> of the encoded message for dealing with extension fields. </param>
/// <param name="blockLength"> of the root message fields. </param>
/// <param name="msgTokens"> in IR format describing the message structure. </param>
/// <param name="listener"> to callback for decoding the primitive values as discovered in the structure. </param>
/// <returns> the index in the underlying buffer after decoding. </returns>
public static int Decode(DirectBuffer buffer, int bufferIndex, int actingVersion, int blockLength,
IList <Token> msgTokens, ITokenListener listener)
{
int groupsBeginIndex = FindNextOrLimit(msgTokens, 1, msgTokens.Count, Signal.BeginGroup);
int varDataSearchStart = groupsBeginIndex != msgTokens.Count ? groupsBeginIndex : 1;
int varDataBeginIndex = FindNextOrLimit(msgTokens, varDataSearchStart, msgTokens.Count, Signal.BeginVarData);
listener.OnBeginMessage(msgTokens[0]);
DecodeFields(buffer, bufferIndex, actingVersion, msgTokens, 0, groupsBeginIndex, listener);
bufferIndex += blockLength;
bufferIndex = DecodeGroups(buffer, bufferIndex, actingVersion, msgTokens, groupsBeginIndex,
varDataBeginIndex, listener);
bufferIndex = DecodeVarData(buffer, bufferIndex, msgTokens, varDataBeginIndex, msgTokens.Count, listener);
listener.OnEndMessage(msgTokens[msgTokens.Count - 1]);
return(bufferIndex);
}
public virtual void OnBitSet(Token fieldToken, DirectBuffer buffer, int bufferIndex, IList <Token> tokens,
int beginIndex, int endIndex, int actingVersion)
{
Token typeToken = tokens[beginIndex + 1];
long encodedValue = ReadEncodingAsLong(buffer, bufferIndex, typeToken, actingVersion);
PrintScope();
Console.Write("{0}:", fieldToken.Name);
for (int i = beginIndex + 1; i < endIndex; i++)
{
Console.Write(" {0}=", tokens[i].Name);
long bitPosition = tokens[i].Encoding.ConstValue.LongValue();
bool flag = (encodedValue & (long)Math.Pow(2, bitPosition)) != 0;
Console.Write(Convert.ToString(flag));
}
Console.WriteLine();
}
public virtual void OnBitSet(Token fieldToken, DirectBuffer buffer, int bufferIndex, IList<Token> tokens,
int beginIndex, int endIndex, int actingVersion)
{
Token typeToken = tokens[beginIndex + 1];
long encodedValue = ReadEncodingAsLong(buffer, bufferIndex, typeToken, actingVersion);
PrintScope();
Console.Write("{0}:", fieldToken.Name);
for (int i = beginIndex + 1; i < endIndex; i++)
{
Console.Write(" {0}=", tokens[i].Name);
long bitPosition = tokens[i].Encoding.ConstValue.LongValue();
bool flag = (encodedValue & (long) Math.Pow(2, bitPosition)) != 0;
Console.Write(Convert.ToString(flag));
}
Console.WriteLine();
}
public void OnEnum(Token fieldToken, DirectBuffer buffer, int bufferIndex, IList <Token> tokens, int beginIndex,
int endIndex, int actingVersion)
{
Token typeToken = tokens[beginIndex + 1];
long encodedValue = ReadEncodingAsLong(buffer, bufferIndex, typeToken, actingVersion);
string value = null;
for (int i = beginIndex + 1; i < endIndex; i++)
{
// TODO to check..
if (encodedValue == tokens[i].Encoding.ConstValue.LongValue())
{
value = tokens[i].Name;
break;
}
}
PrintScope();
Console.WriteLine("{0}={1}", fieldToken.Name, value);
}
public virtual void OnVarData(Token fieldToken, DirectBuffer buffer, int bufferIndex, int length,
Token typeToken)
{
string value;
try
{
int varDataLength = buffer.GetBytes(bufferIndex, _tempBuffer, 0, length);
System.Text.Encoding encoding = System.Text.Encoding.GetEncoding(typeToken.Encoding.CharacterEncoding);
value = encoding.GetString(_tempBuffer, 0, varDataLength);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.Write(ex.StackTrace);
return;
}
PrintScope();
Console.WriteLine("{0}={1}", fieldToken.Name, value);
}
static void WriteObjectToMMF(string mmfFile, object objectData)
{
string mapName = "MyFile";
// Convert .NET object to byte array
byte[] buffer = ObjectToByteArray(objectData);
DirectBuffer _directBuffer = new DirectBuffer(buffer);
Console.WriteLine("size:" + buffer.Length);
using (FileStream fs = new FileStream(mmfFile, FileMode.Create, FileAccess.ReadWrite))
{
fs.SetLength(buffer.Length);
// Create a new memory mapped file
using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(
fs
, mapName
, buffer.Length,
MemoryMappedFileAccess.ReadWrite
, new MemoryMappedFileSecurity()
{
}
, HandleInheritability.Inheritable
, true))
{
//Create a view accessor into the file to accommmodate binary data size
using (MemoryMappedViewAccessor mmfWriter = mmf.CreateViewAccessor(0, buffer.Length))
{
// Write the data
mmfWriter.WriteArray <byte>(0, buffer, 0, buffer.Length);
}
//using (MemoryMappedViewStream mmfWriter = mmf.CreateViewStream(0, buffer.Length))
//{
// // Write the data
// mmfWriter.Write(buffer, 0, buffer.Length);
//}
}
}
}
/// <summary>
/// Deletes up to (but not including) this value for the key.
/// </summary>
/// <param name="tx">transaction</param>
/// <param name="db">database</param>
/// <param name="key">key</param>
/// <param name="value">Value (or part of the value) to which the data will be deleted</param>
/// <param name="firstValue">First value to position to the beginning of dupes. Even if first value is bigger it is OK.</param>
/// <returns></returns>
public static unsafe bool DeleteUpToValue(this Transaction tx, Database db, long key, long value, long firstValue = 0L)
{
int deletedCount = 0;
using (var c = db.OpenCursor(tx))
{
try
{
byte *keyptr = (byte *)Unsafe.AsPointer(ref key);
byte *valptr = (byte *)Unsafe.AsPointer(ref value);
byte *firstvalptr = (byte *)Unsafe.AsPointer(ref firstValue);
var keydb = new DirectBuffer(sizeof(long), keyptr);
var valdb = new DirectBuffer(sizeof(long), firstvalptr);
c.TryFindDup(Lookup.LE, ref keydb, ref valdb);
while (c.Delete(false))
{
deletedCount++;
if (!c.TryGet(ref keydb, ref valdb, CursorGetOption.GetBothRange) || valdb.IsEmpty)
{
break; // empty now
}
var currentValue = valdb.ReadInt64(0);
if (currentValue == value)
{
break;
}
}
return(true);
}
catch (Exception e)
{
throw new DeleteUpToException(
$"Failed after deleting {deletedCount} records."
, e);
}
}
}
public void PutSubFingerprint(SubFingerprintDataDTO subFingerprintDataDTO)
{
var subFingerprintKey = BitConverter.GetBytes(subFingerprintDataDTO.SubFingerprintReference).AsMemory();
var subFingerprintValue = MessagePackSerializer.Serialize(subFingerprintDataDTO, options: serializerOptions).AsMemory();
using (subFingerprintKey.Pin())
{
var subFingerprintKeyBuffer = new DirectBuffer(subFingerprintKey.Span);
using (subFingerprintValue.Pin())
{
var valueBuffer = new DirectBuffer(subFingerprintValue.Span);
databasesHolder.SubFingerprintsDatabase.Put(tx, ref subFingerprintKeyBuffer, ref valueBuffer);
}
var trackKey = BitConverter.GetBytes(subFingerprintDataDTO.TrackReference).AsMemory();
using (trackKey.Pin())
{
var trackKeyBuffer = new DirectBuffer(trackKey.Span);
indexesHolder.TracksSubfingerprintsIndex.Put(tx, ref trackKeyBuffer, ref subFingerprintKeyBuffer);
}
}
}
public static void Main()
{
// This byte array is used for encoding and decoding, this is what ou would send on the wire or save to disk
var byteBuffer = new byte[4096];
// You need to "wrap" the array with a DirectBuffer, this class is used by the generated code to read and write efficiently to the underlying byte array
var directBuffer = new DirectBuffer(byteBuffer);
const short messageTemplateVersion = 0;
int bufferOffset = 0;
// Before encoding a message we need to create a SBE header which specify what we are going to encode (this will allow the decoder to detect that it's an encoded 'car' object)
// We will probably simplify this part soon, so the header gets applied automatically, but for now it's manual
MessageHeader.Wrap(directBuffer, bufferOffset, messageTemplateVersion); // position the MessageHeader on the DirectBuffer, at the correct position
MessageHeader.BlockLength = Car.BlockLength; // size that a car takes on the wire
MessageHeader.SchemaId = Car.SchemaId;
MessageHeader.TemplateId = Car.TemplateId; // identifier for the car object (SBE template ID)
MessageHeader.Version = Car.Schema_Version; // this can be overriden if we want to support different versions of the car object (advanced functionality)
// Now that we have encoded the header in the byte array we can encode the car object itself
bufferOffset += MessageHeader.Size;
Encode(Car, directBuffer, bufferOffset);
// Now we have encoded the message is the byte array, we are going to decode it
// first we decode the header (in a real world scenario you would need the header to decide which SBE decoder you are going to use
bufferOffset = 0;
// position the MessageHeader object at the beginning of the array
MessageHeader.Wrap(directBuffer, bufferOffset, messageTemplateVersion);
// Extract infos from the header
// In a real app you would use that to lookup the applicable flyweight to decode this type of message based on templateId and version.
int actingBlockLength = MessageHeader.BlockLength;
int schemaId = MessageHeader.SchemaId;
int actingVersion = MessageHeader.Version;
bufferOffset += MessageHeader.Size;
// now we decode the message
Decode(Car, directBuffer, bufferOffset, actingBlockLength, schemaId, actingVersion);
}
// ReSharper disable once RedundantAssignment
public bool Write(ref DirectBuffer directBuf)
{
// If the entire array fits in our buffer, copy it as usual.
// Otherwise, switch to direct write from the user-provided buffer
if (_value.Count <= _buf.WriteSpaceLeft)
{
_buf.WriteBytesSimple(_value.Array, _value.Offset, _value.Count);
return(true);
}
if (!_returnedBuffer)
{
directBuf.Buffer = _value.Array;
directBuf.Offset = _value.Offset;
directBuf.Size = _value.Count;
_returnedBuffer = true;
return(false);
}
_returnedBuffer = false;
return(true);
}
private Timestamped <T> Deserialize(DirectBuffer db)
{
Timestamp timestamp = default;
Debug.Assert(!Inner.StreamLog.NoTimestamp);
timestamp = db.Read <Timestamp>(0);
db = db.Slice(Timestamp.Size);
var len = BinarySerializer.Read <T>(Inner.ItemDth, db, out var value);
if (len <= 0)
{
ThrowCannotDeserialize();
}
return(new Timestamped <T>(timestamp, value));
void ThrowCannotDeserialize()
{
throw new InvalidDataException("Cannot deserialize data");
}
}
/// <summary>
/// Reads data from a term in a log buffer.
///
/// If a fragmentsLimit of 0 or less is passed then at least one read will be attempted.
/// </summary>
/// <param name="termBuffer"> to be read for fragments. </param>
/// <param name="offset"> offset within the buffer that the read should begin. </param>
/// <param name="handler"> the handler for data that has been read </param>
/// <param name="fragmentsLimit"> limit the number of fragments read. </param>
/// <param name="header"> to be used for mapping over the header for a given fragment. </param>
/// <param name="errorHandler"> to be notified if an error occurs during the callback. </param>
/// <returns> the number of fragments read </returns>
public static long Read(
DirectBuffer termBuffer,
int offset,
FragmentHandler handler,
int fragmentsLimit,
Header header,
ErrorHandler errorHandler)
{
int fragmentsRead = 0;
int capacity = (int)termBuffer.Length;
try {
do
{
int frameLength = FrameDescriptor.FrameLengthVolatile(termBuffer, offset);
if (frameLength <= 0)
{
break;
}
int termOffset = offset;
offset += BitUtil.Align(frameLength, FrameDescriptor.FRAME_ALIGNMENT);
if (!FrameDescriptor.IsPaddingFrame(termBuffer, termOffset))
{
header.Buffer = termBuffer;
header.Offset = termOffset;
handler?.Invoke(termBuffer, termOffset + DataHeaderFlyweight.HEADER_LENGTH, frameLength - DataHeaderFlyweight.HEADER_LENGTH, header);
++fragmentsRead;
}
}while (fragmentsRead < fragmentsLimit && offset < capacity);
} catch (Exception t) {
errorHandler?.Invoke(t);
}
return(Pack(offset, fragmentsRead));
}
public static void Decode(NewOrderCross sno,
DirectBuffer directBuffer,
int bufferOffset,
int actingBlockLength,
int actingVersion)
{
var buffer = new byte[128];
var sb = new StringBuilder();
// position the car flyweight just after the header on the DirectBuffer
sno.WrapForDecode(directBuffer, bufferOffset, actingBlockLength, actingVersion);
NewOrderCross.NoSidesGroup g = sno.NoSides;
while (g.HasNext)
{
g.Next();
Console.WriteLine(g.Account);
Console.WriteLine(g.ClOrdID);
Console.WriteLine(g.Side);
}
Console.WriteLine(sb.ToString());
}
public async Task <Snapshot> Load(string emitterId)
{
return(env.Read(txn =>
{
using (var owner = MemoryPool <byte> .Shared.Rent(MaxBufferSize))
{
var key = new DirectBuffer(Encoding.UTF8.GetBytes(emitterId));
var data = new DirectBuffer(owner.Memory.Span);
using (var cursor = db.OpenReadOnlyCursor(txn))
{
if (cursor.TryGet(ref key, ref data, CursorGetOption.Last))
{
return (Snapshot)snapshotSerializer.FromBinary(owner.Memory.ToArray(), typeof(Snapshot));
}
else
{
return default(Snapshot);
}
}
}
}));
}
// TODO reuse code, but with yield cursor must be opened inside. Need manual enumerable/enumerator impl
public IEnumerable <KeyValuePair <DirectBuffer, DirectBuffer> > AsEnumerable(Transaction txn)
{
DirectBuffer key = default;
DirectBuffer value = default;
using (var c = OpenReadOnlyCursor(txn))
{
if (c.TryGet(ref key, ref value, CursorGetOption.First))
{
yield return(new KeyValuePair <DirectBuffer, DirectBuffer>(key, value));
value = default;
while (c.TryGet(ref key, ref value, CursorGetOption.NextNoDuplicate))
{
yield return(new KeyValuePair <DirectBuffer, DirectBuffer>(key, value));
value = default;
}
}
}
}
static byte[] GenerateNewOrderSingle()
{
var byteBuffer = new byte[4096];
var directBuffer = new DirectBuffer(byteBuffer);
int bufferOffset = 0;
Type MessageHeaderType = GetType("MessageHeader");
var MessageHeaderObj = GetObjectByType("MessageHeader");
Type SimpleNewOrderType = GetType("SimpleNewOrder");
var SimpleNewOrderObject = GetObjectByType("SimpleNewOrder");
FieldInfo schemaVersion = GetFieldByName("SimpleNewOrder", "SchemaVersion");
MessageHeaderType.InvokeMember("Wrap", BindingFlags.InvokeMethod | BindingFlags.Instance | BindingFlags.Public,
null, MessageHeaderObj, new object[] { directBuffer, bufferOffset, schemaVersion.GetValue(SimpleNewOrderObject) });
FieldInfo size = GetFieldByName("MessageHeader", "Size");
bufferOffset += int.Parse(size.GetValue(MessageHeaderObj).ToString());
SimpleNewOrderType.InvokeMember("WrapForEncode", BindingFlags.InvokeMethod | BindingFlags.Instance | BindingFlags.Public,
null, SimpleNewOrderObject, new object[] { directBuffer, bufferOffset });
PropertyInfo clOrdId = GetPropertyByName("SimpleNewOrder", "ClOrdID");
clOrdId.SetValue(SimpleNewOrderObject, 1234567890);
FieldInfo sizeSimpleNewOrder = GetFieldByName("SimpleNewOrder", "Size");
int SimpleNewOrderLength = int.Parse(sizeSimpleNewOrder.GetValue(SimpleNewOrderObject).ToString()); //NewOrderSingleExample.Encode(sno, directBuffer, bufferOffset);
var byteBuffer2 = new byte[SimpleNewOrderLength];
directBuffer.GetBytes(0, byteBuffer2, 0, SimpleNewOrderLength);
return(byteBuffer);
}
public bool TryFindDup <TKey, TValue>(ReadOnlyTransaction txn, Lookup direction, ref TKey key, ref TValue value)
where TKey : struct where TValue : struct
{
EnsureNoRefs <TKey>();
EnsureNoRefs <TValue>();
var keyPtr = AsPointer(ref key);
var key1 = new DirectBuffer(SizeOf <TKey>(), (nint)keyPtr);
var valuePtr = AsPointer(ref value);
var value1 = new DirectBuffer(SizeOf <TValue>(), (nint)valuePtr);
var res = TryFindDup(txn, direction, ref key1, ref value1);
if (res)
{
key = ReadUnaligned <TKey>(key1.Data);
value = ReadUnaligned <TValue>(value1.Data);
return(true);
}
return(false);
}
public static void Main()
{
Console.WriteLine("\n*** OTF Example ***\n");
// Load a schema (serialized IR) as if we just got it off the wire (you can create a serialized IR schema with SbeTool)
byte[] encodedSchemaBuffer = LoadSchema();
// Encode up a message as if we just got it off the wire
var encodedMsgBuffer = new byte[MsgBufferCapacity];
EncodeTestMessage(encodedMsgBuffer);
// Now lets decode the schema IR so we have IR objects.
//encodedSchemaBuffer.flip();
IntermediateRepresentation ir = DecodeIr(encodedSchemaBuffer);
// From the IR we can create OTF decoder for message headers.
var headerDecoder = new OtfHeaderDecoder(ir.HeaderStructure);
// Now we have IR we can read the message header
int bufferOffset = 0;
var buffer = new DirectBuffer(encodedMsgBuffer);
int templateId = headerDecoder.GetTemplateId(buffer, bufferOffset);
int schemaId = headerDecoder.GetSchemaId(buffer, bufferOffset);
int actingVersion = headerDecoder.GetSchemaVersion(buffer, bufferOffset);
int blockLength = headerDecoder.GetBlockLength(buffer, bufferOffset);
bufferOffset += headerDecoder.Size;
// Given the header information we can select the appropriate message template to do the decode.
// The OTF Java classes are thread safe so the same instances can be reused across multiple threads.
IList <Token> msgTokens = ir.GetMessage(templateId);
bufferOffset = OtfMessageDecoder.Decode(buffer, bufferOffset, actingVersion, blockLength, msgTokens,
new ExampleTokenListener());
}
public override bool Write(ref DirectBuffer directBuf)
{
var bitArray = _value as BitArray;
if (bitArray != null)
{
return(WriteBitArray(bitArray));
}
if (_value is bool)
{
return(WriteBool((bool)_value));
}
var str = _value as string;
if (str != null)
{
return(WriteString(str));
}
throw PGUtil.ThrowIfReached($"Bad type {_value.GetType()} some made its way into BitStringHandler.Write()");
}
/// <summary>
/// Get an integer value from a buffer at a given index.
/// </summary>
/// <param name="buffer"> from which to read. </param>
/// <param name="bufferIndex"> at which he integer should be read. </param>
/// <param name="type"> of the integer encoded in the buffer. </param>
/// <param name="byteOrder"> of the integer in the buffer. </param>
/// <returns> the value of the encoded integer. </returns>
internal static int GetInt(DirectBuffer buffer, int bufferIndex, PrimitiveType type, ByteOrder byteOrder)
{
switch (type.Type)
{
case SbePrimitiveType.Int8:
return buffer.Int8Get(bufferIndex);
case SbePrimitiveType.UInt8:
return buffer.Uint8Get(bufferIndex);
case SbePrimitiveType.Int16:
return byteOrder == ByteOrder.LittleEndian ? buffer.Int16GetLittleEndian(bufferIndex) : buffer.Int16GetBigEndian(bufferIndex);
case SbePrimitiveType.UInt16:
return byteOrder == ByteOrder.LittleEndian ? buffer.Uint16GetLittleEndian(bufferIndex) : buffer.Uint16GetBigEndian(bufferIndex);
case SbePrimitiveType.Int32:
return byteOrder == ByteOrder.LittleEndian ? buffer.Int32GetLittleEndian(bufferIndex) : buffer.Int32GetBigEndian(bufferIndex);
default:
throw new System.ArgumentException("Unsupported type: " + type);
}
}
// ReSharper disable once RedundantAssignment
public override bool Write(ref DirectBuffer directBuf)
{
// If we're back here after having returned a direct buffer, we're done.
if (_returnedBuffer)
{
_returnedBuffer = false;
return(true);
}
// If the entire array fits in our buffer, copy it as usual.
// Otherwise, switch to direct write from the user-provided buffer
if (_value.Count <= _writeBuf.WriteSpaceLeft)
{
_writeBuf.WriteBytes(_value.Array, _value.Offset, _value.Count);
return(true);
}
directBuf.Buffer = _value.Array;
directBuf.Offset = _value.Offset;
directBuf.Size = _value.Count;
_returnedBuffer = true;
return(false);
}
private static string ReadEncodingAsString(DirectBuffer buffer, int index, Token typeToken, int actingVersion)
{
PrimitiveValue constOrNotPresentValue = ConstOrNotPresentValue(typeToken, actingVersion);
if (null != constOrNotPresentValue)
{
return(constOrNotPresentValue.ToString());
}
var sb = new StringBuilder();
Encoding encoding = typeToken.Encoding;
int elementSize = encoding.PrimitiveType.Size;
for (int i = 0, size = typeToken.ArrayLength; i < size; i++)
{
MapEncodingToString(sb, buffer, index + (i * elementSize), encoding);
sb.Append(", ");
}
sb.Length = sb.Length - 2;
return(sb.ToString());
}
public static void Main()
{
Console.WriteLine("\n*** OTF Example ***\n");
// Load a schema (serialized IR) as if we just got it off the wire (you can create a serialized IR schema with SbeTool)
byte[] encodedSchemaBuffer = LoadSchema();
// Encode up a message as if we just got it off the wire
var encodedMsgBuffer = new byte[MsgBufferCapacity];
EncodeTestMessage(encodedMsgBuffer);
// Now lets decode the schema IR so we have IR objects.
//encodedSchemaBuffer.flip();
IntermediateRepresentation ir = DecodeIr(encodedSchemaBuffer);
// From the IR we can create OTF decoder for message headers.
var headerDecoder = new OtfHeaderDecoder(ir.HeaderStructure);
// Now we have IR we can read the message header
int bufferOffset = 0;
var buffer = new DirectBuffer(encodedMsgBuffer);
int templateId = headerDecoder.GetTemplateId(buffer, bufferOffset);
int schemaId = headerDecoder.GetSchemaId(buffer, bufferOffset);
int actingVersion = headerDecoder.GetSchemaVersion(buffer, bufferOffset);
int blockLength = headerDecoder.GetBlockLength(buffer, bufferOffset);
bufferOffset += headerDecoder.Size;
// Given the header information we can select the appropriate message template to do the decode.
// The OTF Java classes are thread safe so the same instances can be reused across multiple threads.
IList<Token> msgTokens = ir.GetMessage(templateId);
bufferOffset = OtfMessageDecoder.Decode(buffer, bufferOffset, actingVersion, blockLength, msgTokens,
new ExampleTokenListener());
}
internal bool Write(WriteBuffer buf, ref DirectBuffer directBuf)
{
Contract.Requires(Statement != null && Statement.All(c => c < 128));
Contract.Requires(Portal != null && Portal.All(c => c < 128));
switch (_state)
{
case State.Header:
var formatCodesSum = InputParameters.Select(p => p.FormatCode).Sum(c => (int)c);
_formatCodeListLength = formatCodesSum == 0 ? 0 : formatCodesSum == InputParameters.Count ? 1 : InputParameters.Count;
var headerLength =
1 + // Message code
4 + // Message length
Portal.Length + 1 +
Statement.Length + 1 +
2; // Number of parameter format codes that follow
if (buf.WriteSpaceLeft < headerLength)
{
Contract.Assume(buf.Size >= headerLength, "Buffer too small for Bind header");
return false;
}
foreach (var c in InputParameters.Select(p => p.LengthCache).Where(c => c != null))
c.Rewind();
var messageLength = headerLength +
2 * _formatCodeListLength + // List of format codes
2 + // Number of parameters
4 * InputParameters.Count + // Parameter lengths
InputParameters.Select(p => p.ValidateAndGetLength()).Sum() + // Parameter values
2 + // Number of result format codes
2 * (UnknownResultTypeList?.Length ?? 1); // Result format codes
buf.WriteByte(Code);
buf.WriteInt32(messageLength-1);
buf.WriteBytesNullTerminated(Encoding.ASCII.GetBytes(Portal));
buf.WriteBytesNullTerminated(Encoding.ASCII.GetBytes(Statement));
buf.WriteInt16(_formatCodeListLength);
_paramIndex = 0;
_state = State.ParameterFormatCodes;
goto case State.ParameterFormatCodes;
case State.ParameterFormatCodes:
// 0 length implicitly means all-text, 1 means all-binary, >1 means mix-and-match
if (_formatCodeListLength == 1)
{
if (buf.WriteSpaceLeft < 2)
return false;
buf.WriteInt16((short)FormatCode.Binary);
}
else if (_formatCodeListLength > 1)
for (; _paramIndex < InputParameters.Count; _paramIndex++)
{
if (buf.WriteSpaceLeft < 2)
return false;
buf.WriteInt16((short)InputParameters[_paramIndex].FormatCode);
}
_state = State.ParameterCount;
goto case State.ParameterCount;
case State.ParameterCount:
if (buf.WriteSpaceLeft < 2)
return false;
buf.WriteInt16(InputParameters.Count);
_paramIndex = 0;
_state = State.ParameterValues;
goto case State.ParameterValues;
case State.ParameterValues:
if (!WriteParameters(buf, ref directBuf))
return false;
_state = State.ResultFormatCodes;
goto case State.ResultFormatCodes;
case State.ResultFormatCodes:
if (UnknownResultTypeList != null)
{
if (buf.WriteSpaceLeft < 2 + UnknownResultTypeList.Length * 2)
return false;
buf.WriteInt16(UnknownResultTypeList.Length);
foreach (var t in UnknownResultTypeList)
buf.WriteInt16(t ? 0 : 1);
}
else
{
if (buf.WriteSpaceLeft < 4)
return false;
buf.WriteInt16(1);
buf.WriteInt16(AllResultTypesAreUnknown ? 0 : 1);
}
_state = State.Done;
return true;
default:
throw PGUtil.ThrowIfReached();
}
}
/// <summary> /// Populates the send buffer with Parse/Describe protocol messages, used for preparing commands /// and for execution in SchemaOnly mode. /// </summary> /// <returns> /// true whether all messages could be populated in the buffer, false otherwise (method needs to be /// called again) /// </returns> bool PopulatePrepare(ref DirectBuffer directBuf) => PopulateParseDescribe(true);
void Send(PopulateMethod populateMethod)
{
while (true)
{
var directBuf = new DirectBuffer();
var completed = populateMethod(ref directBuf);
_connector.SendBuffer();
if (completed)
break; // Sent all messages
// The following is an optimization hack for writing large byte arrays without passing
// through our buffer
if (directBuf.Buffer != null)
{
_connector.WriteBuffer.DirectWrite(directBuf.Buffer, directBuf.Offset, directBuf.Size == 0 ? directBuf.Buffer.Length : directBuf.Size);
directBuf.Buffer = null;
directBuf.Size = 0;
}
if (_writeStatementIndex > 0)
{
// We've send all the messages for the first statement in a multistatement command.
// If we continue blocking writes for the rest of the messages, we risk a deadlock where
// PostgreSQL sends large results for the first statement, while we're sending large
// parameter data for the second. To avoid this, switch to async sends.
// See #641
RemainingSendTask = SendRemaining(populateMethod, CancellationToken.None);
return;
}
}
}
/// <summary>
/// This method is used to asynchronously sends all remaining protocol messages for statements
/// beyond the first one, and *without* waiting for the send to complete. This technique is
/// used to avoid the deadlock described in #641 by allowing the user to read query results
/// while at the same time sending messages for later statements.
/// </summary>
async Task SendRemaining(PopulateMethod populateMethod, CancellationToken cancellationToken)
{
Contract.Requires(_writeStatementIndex > 0);
try
{
while (true)
{
var directBuf = new DirectBuffer();
var completed = populateMethod(ref directBuf);
await _connector.SendBufferAsync(cancellationToken).ConfigureAwait(false);
if (completed)
return; // Sent all messages
// The following is an optimization hack for writing large byte arrays without passing
// through our buffer
if (directBuf.Buffer != null)
{
await _connector.WriteBuffer.DirectWriteAsync(directBuf.Buffer, directBuf.Offset,
directBuf.Size == 0 ? directBuf.Buffer.Length : directBuf.Size, cancellationToken
).ConfigureAwait(false);
directBuf.Buffer = null;
directBuf.Size = 0;
}
}
}
catch (Exception e)
{
Log.Error("Exception while asynchronously sending remaining messages", e, _connector.Id);
}
}
bool PopulateDeallocate(ref DirectBuffer directBuf)
{
Contract.Requires(_connector != null);
var buf = _connector.WriteBuffer;
for (; _writeStatementIndex < _statements.Count; _writeStatementIndex++)
{
var statement = _statements[_writeStatementIndex];
var closeMsg = new CloseMessage(StatementOrPortal.Statement, statement.PreparedStatementName);
if (closeMsg.Length > buf.WriteSpaceLeft)
return false;
closeMsg.WriteFully(buf);
}
if (SyncMessage.Instance.Length > buf.WriteSpaceLeft)
return false;
SyncMessage.Instance.WriteFully(buf);
return true;
}
/// <summary> /// Populates the send buffer with Parse/Describe protocol messages, used for preparing commands /// and for execution in SchemaOnly mode. /// </summary> /// <returns> /// true whether all messages could be populated in the buffer, false otherwise (method needs to be /// called again) /// </returns> bool PopulateExecuteSchemaOnly(ref DirectBuffer directBuf) => PopulateParseDescribe(false);
void SendMessage(FrontendMessage msg)
{
try
{
_log.DebugFormat("Sending: {0}", msg);
var asSimple = msg as SimpleFrontendMessage;
if (asSimple != null)
{
if (asSimple.Length > Buffer.WriteSpaceLeft)
{
Buffer.Flush();
}
Contract.Assume(Buffer.WriteSpaceLeft >= asSimple.Length);
asSimple.Write(Buffer);
return;
}
var asComplex = msg as ChunkingFrontendMessage;
if (asComplex != null)
{
var directBuf = new DirectBuffer();
while (!asComplex.Write(Buffer, ref directBuf))
{
Buffer.Flush();
// The following is an optimization hack for writing large byte arrays without passing
// through our buffer
if (directBuf.Buffer != null)
{
Buffer.Underlying.Write(directBuf.Buffer, 0, directBuf.Size == 0 ? directBuf.Buffer.Length : directBuf.Size);
directBuf.Buffer = null;
directBuf.Size = 0;
}
}
return;
}
throw PGUtil.ThrowIfReached();
}
catch
{
State = ConnectorState.Broken;
throw;
}
}
/// <summary>
/// Populates the send buffer with protocol messages for the execution of prepared statement(s).
/// </summary>
/// <returns>
/// true whether all messages could be populated in the buffer, false otherwise (method needs to be
/// called again)
/// </returns>
bool PopulateExecutePrepared(ref DirectBuffer directBuf)
{
Contract.Requires(_connector != null);
var buf = _connector.WriteBuffer;
for (; _writeStatementIndex < _statements.Count; _writeStatementIndex++)
{
var statement = _statements[_writeStatementIndex];
switch (_sendState)
{
case SendState.Start:
var bind = _connector.BindMessage;
bind.Populate(statement.InputParameters, "", statement.PreparedStatementName);
if (AllResultTypesAreUnknown)
bind.AllResultTypesAreUnknown = AllResultTypesAreUnknown;
else if (_writeStatementIndex == 0 && UnknownResultTypeList != null)
bind.UnknownResultTypeList = UnknownResultTypeList;
_sendState = SendState.Bind;
goto case SendState.Bind;
case SendState.Bind:
if (!_connector.BindMessage.Write(buf, ref directBuf))
return false;
var execute = _connector.ExecuteMessage;
execute.Populate();
_sendState = SendState.Execute;
goto case SendState.Execute;
case SendState.Execute:
execute = _connector.ExecuteMessage;
if (execute.Length > buf.WriteSpaceLeft)
return false;
execute.WriteFully(buf);
_sendState = SendState.Start;
continue;
default:
throw new ArgumentOutOfRangeException($"Invalid state {_sendState} in {nameof(PopulateExecutePrepared)}");
}
}
if (SyncMessage.Instance.Length > buf.WriteSpaceLeft)
return false;
SyncMessage.Instance.WriteFully(buf);
return true;
}
