public async Task ProducerSpeed(int totalMessages, int batchSize, MessageCodec codec)
{
int timeoutInMs = Math.Max(100, totalMessages / 20);
using (var router = await TestConfig.IntegrationOptions.CreateRouterAsync()) {
await router.TemporaryTopicAsync(async topicName => {
var producer = new Producer(router, new ProducerConfiguration(batchSize: totalMessages / 10, batchMaxDelay: TimeSpan.FromMilliseconds(25)));
await producer.UsingAsync(async() => {
var offset = await producer.Router.GetOffsetsAsync(TestConfig.TopicName(), 0, CancellationToken.None);
var maxTimeToRun = TimeSpan.FromMilliseconds(timeoutInMs);
var stopwatch = new Stopwatch();
stopwatch.Start();
var sendList = new List <Task>(totalMessages / batchSize);
var timedOut = Task.Delay(maxTimeToRun);
for (var i = 0; i < totalMessages; i += batchSize)
{
var sendTask = producer.SendAsync(batchSize.Repeat(x => new Message(x.ToString())), offset.topic, offset.partition_id, new SendMessageConfiguration(codec: codec), CancellationToken.None);
sendList.Add(sendTask);
}
var doneSend = Task.WhenAll(sendList.ToArray());
await Task.WhenAny(doneSend, timedOut);
stopwatch.Stop();
if (!doneSend.IsCompleted)
{
var completed = sendList.Count(t => t.IsCompleted);
Assert.Fail($"Only finished sending {completed} of {totalMessages} in {timeoutInMs} ms.");
}
await doneSend;
TestConfig.Log.Info(() => LogEvent.Create($">> done send, time Milliseconds:{stopwatch.ElapsedMilliseconds}"));
});
});
}
}
public void FetchResponse(
[Values(0, 1, 2, 3)] short version,
[Values(0, 1234)] int throttleTime,
[Values("testTopic")] string topicName,
[Values(1, 10)] int topicsPerRequest,
[Values(1, 5)] int totalPartitions,
[Values(MessageCodec.None, MessageCodec.Gzip, MessageCodec.Snappy)] MessageCodec codec,
[Values(
ErrorCode.NONE,
ErrorCode.OFFSET_OUT_OF_RANGE
)] ErrorCode errorCode,
[Values(3)] int messagesPerSet
)
{
#if !DOTNETSTANDARD
if (codec == MessageCodec.Snappy)
{
Assert.Inconclusive($"{codec} is only available in .net core");
}
#endif
var topics = new List <FetchResponse.Topic>();
for (var t = 0; t < topicsPerRequest; t++)
{
var partitionId = t % totalPartitions;
var messages = GenerateMessages(messagesPerSet, (byte)(version >= 2 ? 1 : 0), codec);
topics.Add(new FetchResponse.Topic(topicName + t, partitionId, _randomizer.Next(), errorCode, messages));
}
var response = new FetchResponse(topics, version >= 1 ? TimeSpan.FromMilliseconds(throttleTime) : (TimeSpan?)null);
var responseWithUpdatedAttribute = new FetchResponse(response.responses.Select(t => new FetchResponse.Topic(t.topic, t.partition_id, t.high_watermark, t.error_code,
t.Messages.Select(m => m.Attribute == 0 ? m : new Message(m.Value, m.Key, 0, m.Offset, m.MessageVersion, m.Timestamp)))),
response.throttle_time_ms);
response.AssertCanEncodeDecodeResponse(version, forComparison: responseWithUpdatedAttribute);
}
public ProducerConfig(Int32 throttleSize, Int32 throttleMilliseconds, MessageCodec messageCodec, AcknowlegeStrategy acknowlegeStrategy)
{
ThrottleSize = throttleSize;
ThrottleMilliseconds = throttleMilliseconds;
MessageCodec = messageCodec;
AcknowlegeStrategy = acknowlegeStrategy;
}
public void DecodeMessage_MultipleEscapeCharactersEndingIn_f_x5()
{
byte controllerID = 0x01;
byte msgID = 0x90;
Command cmd = Command.DebugString;
byte[] commandData = Encoding.UTF8.GetBytes("fffff");
byte[] expectedMsg = new byte[] { 0x00, controllerID, msgID, (byte)cmd,
0x66, 0x00, 0x66, 0x00, 0x66, 0x00, 0x66, 0x00, 0x66, 0x00,
0x00, 0x00 };
byte[] expectedMsgNotEscaped = new byte[] { 0x00, controllerID, msgID, (byte)cmd,
0x66, 0x66, 0x66, 0x66, 0x66,
0x00, 0x00 };
expectedMsg[expectedMsg.Length - 2] = MessageCodec.CalculateCRC(expectedMsgNotEscaped);
expectedMsg[expectedMsg.Length - 1] = ETX;
expectedMsg[0] = STX;
byte[] msg = MessageCodec.EncodeMessage(new ProtocolMessage(controllerID, msgID, cmd, commandData));
MessageCodec.DecodeMessages(msg, out List <ProtocolMessage> msgs, out byte[] outputRemainder);
ProtocolMessage myMessage = msgs.First();
Assert.AreEqual(expectedMsg.Length, msg.Length);
CollectionAssert.AreEqual(expectedMsg, msg);
CollectionAssert.AreEqual(commandData, myMessage.CommandData);
}
public void DecodeMessage_OneMessageDifferentStream()
{
byte controllerID = 0x01;
byte msgID = 0x34;
byte cmd = (byte)Command.GetVersion;
byte[] commandData = new byte[] { 0x49, 0x03, 0x92, 0x96, 0x00 };
byte[] inputData = new byte[] { 0x00, controllerID, msgID, cmd, 0x00, 0x00 };
List <byte> dataList = inputData.ToList();
dataList.InsertRange(4, commandData);
inputData = dataList.ToArray();
inputData[inputData.Length - 2] = MessageCodec.CalculateCRC(inputData);
inputData[0] = STX;
inputData[inputData.Length - 1] = ETX;
List <ProtocolMessage> myMsgs = new List <ProtocolMessage>();
MessageCodec.DecodeMessages(inputData.Take(4).ToArray(), out List <ProtocolMessage> msgs, out byte[] outputRemainder);
myMsgs.AddRange(msgs);
MessageCodec.DecodeMessages(inputData.Skip(4).ToArray(), out List <ProtocolMessage> msgs2, out byte[] outputRemainder2, outputRemainder);
myMsgs.AddRange(msgs2);
Assert.IsTrue(myMsgs.Count == 1);
foreach (ProtocolMessage msg in myMsgs)
{
Assert.AreEqual(msg.ControllerID, controllerID);
Assert.AreEqual(msg.MsgID, msgID);
Assert.AreEqual((byte)msg.Command, cmd);
CollectionAssert.AreEqual(commandData, msg.CommandData);
}
CollectionAssert.AreEqual(inputData.Take(4).ToArray(), outputRemainder);
CollectionAssert.AreEqual(new byte[0], outputRemainder2);
}
void Send(UserToken token, byte type, byte request, byte[] message)
{
DataModel model = new DataModel(type, request, message);
byte[] data = MessageCodec.Encode(model);
token.WaitSend(data);
}
/// <summary>
/// Send an enumerable of message objects to a given topic.
/// </summary>
/// <param name="topic">The name of the kafka topic to send the messages to.</param>
/// <param name="messages">The enumerable of messages that will be sent to the given topic.</param>
/// <param name="acks">The required level of acknowlegment from the kafka server. 0=none, 1=writen to leader, 2+=writen to replicas, -1=writen to all replicas.</param>
/// <param name="timeout">Interal kafka timeout to wait for the requested level of ack to occur before returning. Defaults to 1000ms.</param>
/// <param name="codec">The codec to apply to the message collection. Defaults to none.</param>
/// <returns>List of ProduceResponses from each partition sent to or empty list if acks = 0.</returns>
public Task <ProduceResponse[]> SendMessageAsync(string topic, IEnumerable <Message> messages, Int16 acks = 1,
TimeSpan?timeout = null, MessageCodec codec = MessageCodec.CodecNone, int?partition = null)
{
if (_stopToken.IsCancellationRequested)
{
throw new ObjectDisposedException("Cannot send new documents as producer is disposing.");
}
if (timeout == null)
{
timeout = TimeSpan.FromMilliseconds(DefaultAckTimeoutMS);
}
var batch = messages.Select(message => new TopicMessage
{
Acks = acks,
Codec = codec,
Timeout = timeout.Value,
Topic = topic,
Message = message,
Partition = partition
}).ToArray();
_asyncCollection.AddRange(batch);
return(Task.WhenAll(batch.Select(x => x.Tcs.Task)));
}
private void SendConfigButton_Click(object sender, EventArgs e)
{
foreach (ProtocolMessage msg in TemplateProvider.GetConfiguration(0x00, embeddedConfig, 0x01))
{
connector.SendMessage(MessageCodec.EncodeMessage(msg));
}
}
public void ProduceRequest(
[Values(0, 1, 2)] short version,
[Values(0, 2, -1)] short acks,
[Values(0, 1000)] int timeoutMilliseconds,
[Values("testTopic")] string topic,
[Values(1, 10)] int topicsPerRequest,
[Values(1, 5)] int totalPartitions,
[Values(3)] int messagesPerSet,
[Values(MessageCodec.None, MessageCodec.Gzip, MessageCodec.Snappy)] MessageCodec codec)
{
#if !DOTNETSTANDARD
if (codec == MessageCodec.Snappy)
{
Assert.Inconclusive($"{codec} is only available in .net core");
}
#endif
var payloads = new List <ProduceRequest.Topic>();
for (var t = 0; t < topicsPerRequest; t++)
{
var partition = 1 + t % totalPartitions;
payloads.Add(new ProduceRequest.Topic(topic + t, partition, GenerateMessages(messagesPerSet, (byte)(version >= 2 ? 1 : 0), codec), codec));
}
var request = new ProduceRequest(payloads, TimeSpan.FromMilliseconds(timeoutMilliseconds), acks);
var requestWithUpdatedAttribute = new ProduceRequest(request.topics.Select(t => new ProduceRequest.Topic(t.topic, t.partition_id,
t.Messages.Select(m => m.Attribute == 0 ? m : new Message(m.Value, m.Key, 0, m.Offset, m.MessageVersion, m.Timestamp)))),
request.timeout, request.acks);
request.AssertCanEncodeDecodeRequest(version, forComparison: requestWithUpdatedAttribute);
}
private void DispatchQueryRegisterMessage(ProtocolMessage protocolMessage)
{
QueryRegisterMessage response = new QueryRegisterMessage(protocolMessage);
if (response.Value == null)
{
// Error reading occured!
throw new ArgumentException("Error reading occured");
}
if (this.Nodes.All(x => x.Id != protocolMessage.ControllerID))
{
throw new ArgumentException("No node found for the msg");
}
CpuNode node = this.Nodes.First(x => x.Id == protocolMessage.ControllerID);
MessageCodec.GetControlByteValues(node.ProtocolVersion, response.Control, out ReadWrite readWrite, out Source source, out int derefDepth);
Register r = node.EmbeddedConfig.GetRegister(response.Offset, readWrite);
if (r == null)
{
throw new ArgumentException("No register found with this offset or readwrite");
}
NewValueReceived(this, new ValueReceivedEventArgs(RegisterValue.GetRegisterValueByVariableType(r.VariableType, response.Value), r, node));
}
public override void SetupSignalTracing(CpuNode cpuNode, Register register, ChannelMode channelMode)
{
// Make sure the CpuNode is present in our dictionary
if (DebugChannels.ContainsKey(cpuNode))
{
if (!this.DebugChannels[cpuNode].ContainsValue(register) && channelMode != ChannelMode.Off)
{
for (byte i = 0; i < 255; i++)
{
if (!register.CpuNode.DebugChannels.ContainsKey(i))
{
this.DebugChannels[cpuNode].Add(i, register);
break;
}
}
}
if (this.DebugChannels[cpuNode].ContainsValue(register))
{
Version protocolVersion = cpuNode.ProtocolVersion;
ConfigChannelMessage msg = new ConfigChannelMessage()
{
ChannelId = this.DebugChannels[cpuNode].First(x => x.Value.Id == register.Id).Key,
Mode = channelMode,
Offset = register.Offset,
Control = MessageCodec.GetControlByte(protocolVersion, register.ReadWrite, register.Source, register.DerefDepth),
Size = register.Size,
};
SendMessage(cpuNode.Id, msg);
if (this.DebugChannels[cpuNode].ContainsValue(register) && channelMode == ChannelMode.Off)
{
this.DebugChannels[cpuNode].Remove(this.DebugChannels[cpuNode].First(x => x.Value.Id == register.Id).Key);
}
}
}
}
public void DecodeMessage_TwoMessagesSameStream_GarbageInBetween()
{
byte controllerID = 0x01;
byte msgID = 0x34;
byte cmd = (byte)Command.GetVersion;
byte[] commandData = new byte[] { 0x49, 0x03, 0x92, 0x96, 0x00 };
byte[] inputData = new byte[] { 0x00, controllerID, msgID, cmd, 0x00, 0x00 };
List <byte> dataList = inputData.ToList();
dataList.InsertRange(4, commandData);
inputData = dataList.ToArray();
inputData[inputData.Length - 2] = MessageCodec.CalculateCRC(inputData);
inputData[0] = STX;
inputData[inputData.Length - 1] = ETX;
dataList = inputData.ToList();
dataList.AddRange(new byte[] { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF });
dataList.AddRange(inputData);
inputData = dataList.ToArray();
MessageCodec.DecodeMessages(inputData, out List <ProtocolMessage> msgs, out byte[] outputRemainder);
Assert.IsTrue(msgs.Count == 2);
foreach (ProtocolMessage msg in msgs)
{
Assert.AreEqual(msg.ControllerID, controllerID);
Assert.AreEqual(msg.MsgID, msgID);
Assert.AreEqual((byte)msg.Command, cmd);
CollectionAssert.AreEqual(commandData, msg.CommandData);
}
CollectionAssert.AreEqual(new byte[0], outputRemainder);
}
public void DecodeMessage_OneMessageCommandNoData_Remainder()
{
byte controllerID = 0x01;
byte msgID = 0x34;
byte cmd = (byte)Command.GetVersion;
byte[] commandData = new byte[0];
byte[] inputData = new byte[] { 0x00, controllerID, msgID, cmd, 0x00, 0x00 };
List <byte> dataList = inputData.ToList();
dataList.InsertRange(4, commandData);
inputData = dataList.ToArray();
inputData[inputData.Length - 2] = MessageCodec.CalculateCRC(inputData);
inputData[0] = STX;
inputData[inputData.Length - 1] = ETX;
dataList = inputData.ToList();
dataList.AddRange(new byte[] { 0xFF, 0xFF, 0xFF, 0xFF });
inputData = dataList.ToArray();
MessageCodec.DecodeMessages(inputData, out List <ProtocolMessage> msgs, out byte[] outputRemainder);
Assert.IsTrue(msgs.Count == 1);
ProtocolMessage msg = msgs.First();
Assert.AreEqual(msg.ControllerID, controllerID);
Assert.AreEqual(msg.MsgID, msgID);
Assert.AreEqual((byte)msg.Command, cmd);
CollectionAssert.AreEqual(commandData, msg.CommandData);
CollectionAssert.AreEqual(new byte[] { 0xFF, 0xFF, 0xFF, 0xFF }, outputRemainder);
}
/// <summary>
/// 发送消息到队列
/// </summary>
/// <param name="topic"></param>
/// <param name="datas"></param>
/// <param name="acks"></param>
/// <param name="timeout"></param>
/// <param name="codec"></param>
public void Pub(string topic, List <string> datas, short acks = 1, TimeSpan?timeout = default(TimeSpan?),
MessageCodec codec = MessageCodec.CodecNone)
{
var msgs = datas.Select(item => new Message(item)).ToList();
_producer.SendMessageAsync(topic, msgs, acks, timeout, codec);
}
public void UseCodec(MessageCodec codec)
{
if (codec == MessageCodec.Snappy)
{
throw new ArgumentOutOfRangeException("codec", "Snappy not support yet");
}
_config.MessageCodec = codec;
}
/// <summary>
/// Initializes a new channel instance
/// </summary>
/// <param name="manager">
/// Containing channel listener/factory
/// </param>
/// <param name="codec">
/// The channel message coder/decoder
/// </param>
/// <param name="remoteAddress">
/// The output address
/// </param>
/// <param name="socket">
/// The UDP channel socket
/// </param>
public OutputChannel(
ChannelManagerBase manager,
MessageCodec codec,
EndpointAddress remoteAddress,
UdpSocket socket)
: base(manager, codec, remoteAddress)
{
this.socket = socket;
}
/// <summary>
/// Initializes a new channel instance
/// </summary>
/// <param name="manager">
/// Containing channel listener/factory
/// </param>
/// <param name="codec">
/// The channel message coder/decoder
/// </param>
/// <param name="localAddress">
/// The input address
/// </param>
/// <param name="socket">
/// The datagram socket for this channel
/// </param>
public ReplyChannel(
ChannelManagerBase manager,
MessageCodec codec,
EndpointAddress localAddress,
UdpSocket socket)
: base(manager, codec, localAddress)
{
this.socket = socket;
}
/// <summary>
/// Initializes a new request context instance
/// </summary>
/// <param name="request">
/// The original request message
/// </param>
/// <param name="codec">
/// The channel message coder/decoder
/// </param>
/// <param name="socket">
/// The datagram socket for this channel
/// </param>
/// <param name="replyEP">
/// The address/port of the client endpoint to send to
/// </param>
public RequestReply(
Message request,
MessageCodec codec,
UdpSocket socket,
EndPoint replyEP)
: base(request, codec)
{
this.socket = socket;
this.clientEndpoint = replyEP;
}
public override void WriteConsole(CpuNode cpuNode, string message)
{
DebugStringMessage msg = new DebugStringMessage()
{
Message = message
};
// TODO add the msgID
this.SendMessage(MessageCodec.EncodeMessage(msg.ToProtocolMessage(cpuNode.Id, 0x01)));
}
/// <summary>
/// Initializes a new channel instance
/// </summary>
/// <param name="manager">
/// Containing channel listener/factory
/// </param>
/// <param name="codec">
/// The channel message coder/decoder
/// </param>
/// <param name="remoteAddress">
/// The output address
/// </param>
/// <param name="socket">
/// The UDP channel socket
/// </param>
public RequestChannel(
ChannelManagerBase manager,
MessageCodec codec,
EndpointAddress remoteAddress,
UdpSocket socket)
: base(manager, codec, remoteAddress)
{
this.socket = socket;
this.requestMap = new Dictionary<System.Xml.UniqueId, PendingRequest>();
this.pending = 0;
}
private void SendSine()
{
sendSine = true;
long currentTime = stopwatch.ElapsedMilliseconds;
while (stopwatch.ElapsedMilliseconds < currentTime + 600000 && sendSine)
{
double value = (((double)stopwatch.ElapsedMilliseconds / 100) % 6.28 - 3.14);
connector.SendMessage(MessageCodec.EncodeMessage(TemplateProvider.GetChannelDataMessage((int)(100 * Math.Sin(value) + 128), stopwatch.ElapsedMilliseconds)));
Thread.Sleep(1);
}
}
public CommandMessageWorker() :
base(MonitorAdress, ZSocketType.ROUTER, MessagingCoreSettings.Default.CommandSocketReceivePollRate)
{
RecvBuffer = new TransformBlock <Object, MessageData>(
(msg) => MessageCodec.DecodeMessage((ZMessage)msg));
SendBuffer = new TransformBlock <(Int32 NodeId, IList <Byte[]> Msg), Object>(
(msg) => MessageCodec.EncodeCommandMessage(msg.NodeId, msg.Msg));
m_sendTransform = new TransformBlock <Object, ZMessage>(obj => (ZMessage)obj);
SendBuffer.LinkTo(m_sendTransform);
m_sendTransform.LinkTo(m_sendAction);
m_receiveBuffer.LinkTo(RecvBuffer);
}
public override void QueryValue(CpuNode cpuNode, Register register)
{
Version protocolVersion = cpuNode.ProtocolVersion;
byte control = MessageCodec.GetControlByte(protocolVersion, register.ReadWrite, register.Source, register.DerefDepth);
QueryRegisterMessage msg = new QueryRegisterMessage()
{
Offset = register.Offset,
Control = control,
};
// TODO add the msgID
this.SendMessage(MessageCodec.EncodeMessage(msg.ToProtocolMessage(cpuNode.Id, 0x01)));
}
public void TestEmpty()
{
var message = new Message(new Dictionary <string, string>()
{
}, new byte[] {});
var encoder = new MessageCodec();
var encoded = encoder.Encode(message);
var decoded = encoder.Decode(encoded);
Assert.AreEqual(decoded.Headers.Count, 0);
Assert.AreEqual(decoded.Payload.Length, 0);
}
public void EncodeMessage_CommandDataEscapeCharactersForSTXETX()
{
ProtocolMessage msg = new ProtocolMessage(STX, ETX, Command.ConfigChannel);
byte[] output = MessageCodec.EncodeMessage(msg);
byte crc = output[output.Length - 2];
Assert.AreEqual(8, output.Length);
Assert.AreEqual(ESC, output[1]);
Assert.AreEqual(STX, (byte)(ESC ^ output[2]));
Assert.AreEqual(ESC, output[3]);
Assert.AreEqual(ETX, (byte)(ESC ^ output[4]));
}
public void DecodeMessage_DecodeEncodedMessageDataEscaped()
{
ProtocolMessage msg = new ProtocolMessage(STX, 0x02, Command.ConfigChannel, new byte[] { ESC, STX, ETX });
byte[] encodedMsg = MessageCodec.EncodeMessage(msg);
MessageCodec.DecodeMessages(encodedMsg, out List <ProtocolMessage> msgs, out byte[] remainder);
Assert.IsTrue(msgs.Count == 1);
Assert.IsTrue(remainder.Length == 0);
Assert.AreEqual(msg.ControllerID, msgs.First().ControllerID);
Assert.AreEqual(msg.MsgID, msgs.First().MsgID);
Assert.AreEqual(msg.Command, msgs.First().Command);
CollectionAssert.AreEqual(msg.CommandData, msgs.First().CommandData);
}
/// <summary>
/// Send a enumerable of message objects to a given topic.
/// </summary>
/// <param name="messages">The enumerable of messages that will be sent to the given topic. All messages *MUST* have a topic assigned to them.</param>
/// <param name="acks">The required level of acknowlegment from the kafka server. 0=none, 1=writen to leader, 2+=writen to replicas, -1=writen to all replicas.</param>
/// <param name="timeoutMS">Interal kafka timeout to wait for the requested level of ack to occur before returning.</param>
/// <param name="codec">The codec to apply to the message collection. Defaults to none.</param>
/// <returns>List of ProduceResponses for each message sent or empty list if acks = 0.</returns>
public async Task<List<ProduceResponse>> SendMultiTopicMessagesAsync(IEnumerable<Message> messages, Int16 acks = 1, int timeoutMS = 1000, MessageCodec codec = MessageCodec.CodecNone)
{
Interlocked.Increment(ref _currentAsyncQueue);
try
{
//This goes against async philosophy but it convenient for dataflow management
while (_maximumAsyncQueue != -1 && _currentAsyncQueue >= _maximumAsyncQueue)
{
Thread.Sleep(100);
}
//group message by the server connection they will be sent to
var routeGroup = from message in messages
select new {Route = _router.SelectBrokerRoute(message.Topic, message.Key), Message = message}
into routes
group routes by routes.Route;
var sendTasks = new List<Task<List<ProduceResponse>>>();
foreach (var route in routeGroup)
{
var request = new ProduceRequest
{
Acks = acks,
TimeoutMS = timeoutMS,
Payload = new List<Payload>
{
new Payload
{
Codec = codec,
Topic = route.Key.Topic,
Partition = route.Key.PartitionId,
Messages = route.Select(x => x.Message).ToList()
}
}
};
sendTasks.Add(route.Key.Connection.SendAsync(request));
}
await Task.WhenAll(sendTasks.ToArray());
return sendTasks.SelectMany(t => t.Result).ToList();
}
finally
{
Interlocked.Decrement(ref _currentAsyncQueue);
}
}
U2FHidDeviceInfo OnInitAnswered(FidoU2FHidMessage response, ArraySegment <byte> requestNonce)
{
log.Info("Initialization response received");
ArraySegment <byte> responseNonce;
var deviceInfo = MessageCodec.DecodeInitResponse(response.Data, out responseNonce);
if (!responseNonce.ContentEquals(requestNonce))
{
throw new Exception("Invalid nonce, not an answer to our init request");
}
DeviceInfo = deviceInfo;
return(deviceInfo);
}
protected void ReceiveMessage(BytesReceivedEventArgs e)
{
MessageCodec.DecodeMessages(e.Message, out List <ProtocolMessage> messages, out this.remainderBytes, this.remainderBytes);
foreach (ProtocolMessage protocolMessage in messages)
{
switch (protocolMessage.Command)
{
case Command.GetVersion:
this.DispatchVersionMessage(protocolMessage);
break;
case Command.GetInfo:
break;
case Command.WriteRegister:
break;
case Command.QueryRegister:
this.DispatchQueryRegisterMessage(protocolMessage);
break;
case Command.ConfigChannel:
break;
case Command.Decimation:
break;
case Command.ResetTime:
break;
case Command.ReadChannelData:
DispatchReadChannelDataMessage(protocolMessage);
break;
case Command.DebugString:
break;
case Command.EmbeddedConfiguration:
break;
case Command.Tracing:
break;
default:
throw new ArgumentOutOfRangeException();
}
}
}
/// <summary>
/// Send a enumerable of message objects to a given topic.
/// </summary>
/// <param name="topic">The name of the kafka topic to send the messages to.</param>
/// <param name="messages">The enumerable of messages that will be sent to the given topic.</param>
/// <param name="acks">The required level of acknowlegment from the kafka server. 0=none, 1=writen to leader, 2+=writen to replicas, -1=writen to all replicas.</param>
/// <param name="timeout">Interal kafka timeout to wait for the requested level of ack to occur before returning. Defaults to 1000ms.</param>
/// <param name="codec">The codec to apply to the message collection. Defaults to none.</param>
/// <returns>List of ProduceResponses for each message sent or empty list if acks = 0.</returns>
public async Task<List<ProduceResponse>> SendMessageAsync(string topic, IEnumerable<Message> messages, Int16 acks = 1,
TimeSpan? timeout = null, MessageCodec codec = MessageCodec.CodecNone)
{
if (timeout == null)
timeout = TimeSpan.FromMilliseconds(DefaultTimeoutMS);
try
{
_sendSemaphore.Wait();
//group message by the server connection they will be sent to
var routeGroup = from message in messages
select new { Route = _router.SelectBrokerRoute(topic, message.Key), Message = message }
into routes
group routes by routes.Route;
var sendTasks = new List<Task<List<ProduceResponse>>>();
foreach (var route in routeGroup)
{
var request = new ProduceRequest
{
Acks = acks,
TimeoutMS = (int)timeout.Value.TotalMilliseconds,
Payload = new List<Payload>
{
new Payload
{
Codec = codec,
Topic = route.Key.Topic,
Partition = route.Key.PartitionId,
Messages = route.Select(x => x.Message).ToList()
}
}
};
sendTasks.Add(route.Key.Connection.SendAsync(request));
}
await Task.WhenAll(sendTasks.ToArray());
return sendTasks.SelectMany(t => t.Result).ToList();
}
finally
{
_sendSemaphore.Release();
}
}
public void TestAllocate()
{
var codec = new MessageCodec(manager, wcfCodec, 65536);
using (var buffer1 = codec.Allocate())
{
Assert.IsNotNull(buffer1.Raw);
Assert.AreEqual(buffer1.Offset, 0);
Assert.AreEqual(buffer1.Length, 65536);
using (var buffer2 = codec.Allocate())
{
Assert.IsNotNull(buffer2.Raw);
Assert.AreEqual(buffer2.Offset, 0);
Assert.AreEqual(buffer2.Length, 65536);
Assert.AreNotEqual(buffer1.Raw, buffer2.Raw);
}
}
}
public void TestNoHeaders()
{
var message = new Message(new Dictionary <string, string>()
{
}, new byte[] { 1, 2, 3 });
var encoder = new MessageCodec();
var encoded = encoder.Encode(message);
var decoded = encoder.Decode(encoded);
Assert.AreEqual(decoded.Headers.Count, 0);
Assert.AreEqual(decoded.Payload.Length, 3);
Assert.AreEqual(decoded.Payload[0], 1);
Assert.AreEqual(decoded.Payload[1], 2);
Assert.AreEqual(decoded.Payload[2], 3);
}
private IEnumerable <Message> GenerateMessages(int count, byte version, MessageCodec codec = MessageCodec.None)
{
var random = new Random(42);
var messages = new List <Message>();
for (var m = 0; m < count; m++)
{
var key = m > 0 ? new byte[8] : null;
var value = new byte[8 * (m + 1)];
if (key != null)
{
random.NextBytes(key);
}
random.NextBytes(value);
messages.Add(new Message(new ArraySegment <byte>(value), key != null ? new ArraySegment <byte>(key) : new ArraySegment <byte>(), (byte)codec, version: version, timestamp: version > 0 ? DateTimeOffset.UtcNow : (DateTimeOffset?)null));
}
return(messages);
}
public void EncodeMessage_NoCommandDataValidFalse_ByteArray()
{
ProtocolMessage msg = new ProtocolMessage(0x01, 0x84, Command.QueryRegister);
byte[] output = MessageCodec.EncodeMessage(msg);
byte crc = output[output.Length - 2];
Assert.AreEqual(output[0], STX);
Assert.AreEqual(output[1], msg.ControllerID);
Assert.AreEqual(output[2], msg.MsgID);
Assert.AreEqual(output[3], (byte)msg.Command);
Assert.AreEqual(output[output.Length - 1], ETX);
Assert.IsTrue(string.IsNullOrEmpty(MessageCodec.ValidateMessageBytes(output)));
output[0] = 0x00;
output[output.Length - 2] = 0x00;
output[output.Length - 1] = 0x00;
Assert.AreEqual(crc, MessageCodec.CalculateCRC(output));
}
public void DecodeMessage_OneMessage_UnexpectedSTX()
{
byte controllerID = 0x01;
byte msgID = 0x34;
byte cmd = (byte)Command.GetVersion;
byte[] commandData = new byte[] { 0x49, STX, 0x67, 0x96, 0x00 };
byte[] inputData = new byte[] { 0x00, controllerID, msgID, cmd, 0x00, 0x00 };
List <byte> dataList = inputData.ToList();
dataList.InsertRange(4, commandData);
inputData = dataList.ToArray();
inputData[inputData.Length - 2] = MessageCodec.CalculateCRC(inputData);
inputData[0] = STX;
inputData[inputData.Length - 1] = ETX;
MessageCodec.DecodeMessages(inputData, out List <ProtocolMessage> msgs, out byte[] outputRemainder);
Assert.IsFalse(msgs.First().Valid);
}
private static MTProtoClientBuilder CreateDefault()
{
var clientTransportFactory = new ClientTransportFactory();
var tlRig = new TLRig();
var messageIdGenerator = new MessageIdGenerator();
var hashServices = new HashServices();
var encryptionServices = new EncryptionServices();
var randomGenerator = new RandomGenerator();
var messageCodec = new MessageCodec(tlRig, hashServices, encryptionServices, randomGenerator);
var keyChain = new KeyChain(tlRig, hashServices);
var nonceGenerator = new NonceGenerator();
return new MTProtoClientBuilder(clientTransportFactory,
tlRig,
messageIdGenerator,
messageCodec,
hashServices,
encryptionServices,
nonceGenerator,
keyChain);
}
public void DecodeMessage_OneMessageNoCommandData_NoRemainder()
{
byte controllerID = 0x01;
byte msgID = 0x34;
byte cmd = (byte)Command.GetVersion;
byte[] inputData = new byte[] { 0x00, controllerID, msgID, cmd, 0x00, 0x00 };
inputData[inputData.Length - 2] = MessageCodec.CalculateCRC(inputData);
inputData[0] = STX;
inputData[inputData.Length - 1] = ETX;
MessageCodec.DecodeMessages(inputData, out List <ProtocolMessage> msgs, out byte[] outputRemainder);
Assert.IsTrue(msgs.Count == 1);
ProtocolMessage msg = msgs.First();
Assert.AreEqual(msg.ControllerID, controllerID);
Assert.AreEqual(msg.MsgID, msgID);
Assert.AreEqual((byte)msg.Command, cmd);
CollectionAssert.AreEqual(new byte[0], outputRemainder);
}
/// <summary>
/// Send an enumerable of message objects to a given topic.
/// </summary>
/// <param name="topic">The name of the kafka topic to send the messages to.</param>
/// <param name="messages">The enumerable of messages that will be sent to the given topic.</param>
/// <param name="acks">The required level of acknowlegment from the kafka server. 0=none, 1=writen to leader, 2+=writen to replicas, -1=writen to all replicas.</param>
/// <param name="timeout">Interal kafka timeout to wait for the requested level of ack to occur before returning. Defaults to 1000ms.</param>
/// <param name="codec">The codec to apply to the message collection. Defaults to none.</param>
/// <returns>List of ProduceResponses from each partition sent to or empty list if acks = 0.</returns>
public async Task<List<ProduceResponse>> SendMessageAsync(string topic, IEnumerable<Message> messages, Int16 acks = 1,
TimeSpan? timeout = null, MessageCodec codec = MessageCodec.CodecNone)
{
if (_stopToken.IsCancellationRequested) throw new ObjectDisposedException("Cannot send new documents as producer is disposing.");
if (timeout == null) timeout = TimeSpan.FromMilliseconds(DefaultAckTimeoutMS);
var batch = messages.Select(message => new TopicMessage
{
Acks = acks,
Codec = codec,
Timeout = timeout.Value,
Topic = topic,
Message = message
}).ToList();
await _nagleBlockingCollection.AddRangeAsync(batch, _stopToken.Token).ConfigureAwait(false);
var results = new List<ProduceResponse>();
foreach (var topicMessage in batch)
{
results.Add(await topicMessage.Tcs.Task.ConfigureAwait(false));
}
return results.Distinct().ToList();
}
public void TestEncodeDecode()
{
var codec = new MessageCodec(manager, wcfCodec, 65536);
// invalid encode
try
{
codec.Encode(null);
Assert.Fail("Expected: Exception");
}
catch (AssertFailedException) { throw; }
catch { }
try
{
codec.Encode(null, null);
Assert.Fail("Expected: Exception");
}
catch (AssertFailedException) { throw; }
catch { }
try
{
codec.Encode(CreateMessage(1), null);
Assert.Fail("Expected: Exception");
}
catch (AssertFailedException) { throw; }
catch { }
try
{
codec.Encode(null, new MemoryStream());
Assert.Fail("Expected: Exception");
}
catch (AssertFailedException) { throw; }
catch { }
// invalid decode
Assert.AreEqual(codec.Decode((Byte[])null), null);
Assert.AreEqual(codec.Decode(new Byte[0]), null);
try
{
codec.Decode(new Byte[1024]);
Assert.Fail("Expected: Exception");
}
catch (AssertFailedException) { throw; }
catch { }
Assert.AreEqual(codec.Decode(default(ArraySegment<Byte>)), null);
try
{
codec.Decode(new ArraySegment<Byte>(new Byte[1024]));
Assert.Fail("Expected: Exception");
}
catch (AssertFailedException) { throw; }
catch { }
Assert.AreEqual(codec.Decode((Stream)null), null);
Assert.AreEqual(codec.Decode(new MemoryStream()), null);
try
{
var stream = new MemoryStream();
stream.Write(new Byte[1024], 0, 1024);
stream.Position = 0;
codec.Decode(stream);
Assert.Fail("Expected: Exception");
}
catch (AssertFailedException) { throw; }
catch { }
// valid buffered message
AssertIsEqualMessage(
CreateMessage(1),
codec.Decode(codec.Encode(CreateMessage(1)))
);
var encoded = codec.Encode(CreateMessage(1));
AssertIsEqualMessage(
CreateMessage(1),
codec.Decode(encoded.Raw.Skip(encoded.Offset).Take(encoded.Length).ToArray())
);
// valid streamed message
using (var stream = new MemoryStream())
{
codec.Encode(CreateMessage(2), stream);
stream.Position = 0;
AssertIsEqualMessage(CreateMessage(2), codec.Decode(stream));
}
// using encoded
codec.UsingEncoded(
CreateMessage(3),
buffer1 =>
{
var buffer2 = codec.Allocate();
Array.Copy(buffer1.Raw, buffer1.Offset, buffer2.Raw, buffer2.Offset, buffer1.Length);
AssertIsEqualMessage(
CreateMessage(3),
codec.Decode(new ArraySegment<Byte>(buffer2.Raw, buffer2.Offset, buffer1.Length))
);
}
);
}
/// <summary>
/// Send a enumerable of message objects to a given topic.
/// </summary>
/// <param name="topic">The name of the kafka topic to send the messages to.</param>
/// <param name="messages">The enumerable of messages that will be sent to the given topic.</param>
/// <param name="acks">The required level of acknowlegment from the kafka server. 0=none, 1=writen to leader, 2+=writen to replicas, -1=writen to all replicas.</param>
/// <param name="timeoutMS">Interal kafka timeout to wait for the requested level of ack to occur before returning.</param>
/// <param name="codec">The codec to apply to the message collection. Defaults to none.</param>
/// <returns>List of ProduceResponses for each message sent or empty list if acks = 0.</returns>
public Task<List<ProduceResponse>> SendMessageAsync(string topic, IEnumerable<Message> messages, Int16 acks = 1, int timeoutMS = 1000, MessageCodec codec = MessageCodec.CodecNone)
{
// with the addition of method SendMultiTopicMessagesAsync,
// to keep things DRY all we have to do here is assign the topic to the message
var messageWithTopic = messages.Select(m =>
{
m.Topic = topic;
return m;
});
return SendMultiTopicMessagesAsync(messageWithTopic, acks, timeoutMS, codec);
}
public async void ProducesShouldSendExpectedProduceRequestForEachCodecCombination(MessageCodec codec1, MessageCodec codec2, int expected)
{
var routerProxy = new FakeBrokerRouter();
var producer = new Producer(routerProxy.Create()) { BatchSize = 100 };
using (producer)
{
var calls = new[]
{
producer.SendMessageAsync(FakeBrokerRouter.TestTopic, new[] {new Message(), new Message()}, codec: codec1),
producer.SendMessageAsync(FakeBrokerRouter.TestTopic, new[] {new Message(), new Message()}, codec: codec2)
};
await Task.WhenAll(calls);
Assert.That(routerProxy.BrokerConn0.ProduceRequestCallCount, Is.EqualTo(expected));
Assert.That(routerProxy.BrokerConn1.ProduceRequestCallCount, Is.EqualTo(expected));
}
}
/// <summary>
/// Send an enumerable of message objects to a given topic.
/// </summary>
/// <param name="topic">The name of the kafka topic to send the messages to.</param>
/// <param name="messages">The enumerable of messages that will be sent to the given topic.</param>
/// <param name="acks">The required level of acknowlegment from the kafka server. 0=none, 1=writen to leader, 2+=writen to replicas, -1=writen to all replicas.</param>
/// <param name="timeout">Interal kafka timeout to wait for the requested level of ack to occur before returning. Defaults to 1000ms.</param>
/// <param name="codec">The codec to apply to the message collection. Defaults to none.</param>
/// <returns>List of ProduceResponses from each partition sent to or empty list if acks = 0.</returns>
public Task<ProduceResponse[]> SendMessageAsync(string topic, IEnumerable<Message> messages, Int16 acks = 1,
TimeSpan? timeout = null, MessageCodec codec = MessageCodec.CodecNone, int? partition = null)
{
if (_stopToken.IsCancellationRequested)
throw new ObjectDisposedException("Cannot send new documents as producer is disposing.");
if (timeout == null) timeout = TimeSpan.FromMilliseconds(DefaultAckTimeoutMS);
var batch = messages.Select(message => new TopicMessage
{
Acks = acks,
Codec = codec,
Timeout = timeout.Value,
Topic = topic,
Message = message,
Partition = partition
}).ToArray();
_asyncCollection.AddRange(batch);
return Task.WhenAll(batch.Select(x => x.Tcs.Task));
}
