/// <summary> /// Consumes messages from Kafka. /// </summary> /// <param name="request">The request to send to Kafka.</param> /// <returns>A list of messages from Kafka.</returns> public List <Message> Consume(ConsumerRequest request) { List <Message> messages = new List <Message>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); // TODO: need to check in on kafka error codes...assume all's good for now byte[] unbufferedData = data.Skip(2).ToArray(); int processed = 0; int length = unbufferedData.Length - 4; int messageSize = 0; while (processed <= length) { messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray <byte>()), 0); messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray <byte>())); processed += 4 + messageSize; } } } return(messages); }
public static KafkaConnection GetConnection(string host, int port) { KafkaConnection kafkaConnection = null; lock (locker) { string connectionKey = GetConnectionKey(host, port); KafkaServerConnectionPool serverConnections = null; if (kafkaServerConnections.ContainsKey(connectionKey)) { kafkaConnection = kafkaServerConnections[connectionKey].GetConnection(); } else { serverConnections = new KafkaServerConnectionPool(_maxPoolSize, _lifespan, host, port, _bufferSize, _socketTimeout, _idleTimeToKeepAlive, _keepAliveInterval, _socketPollingIntervalTimeout, _socketPollingLevel); kafkaConnection = serverConnections.GetConnection(); kafkaServerConnections[connectionKey] = serverConnections; } } return(kafkaConnection); }
/// <summary> /// Get a list of valid offsets (up to maxSize) before the given time. /// </summary> /// <param name="request">The offset request.</param> /// <returns>List of offsets, in descending order.</returns> public IList <long> GetOffsetsBefore(OffsetRequest request) { List <long> offsets = new List <long>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); // TODO: need to check in on kafka error codes...assume all's good for now byte[] unbufferedData = data.Skip(2).ToArray(); // first four bytes are the number of offsets int numOfOffsets = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray <byte>()), 0); int position = 0; for (int ix = 0; ix < numOfOffsets; ix++) { position = (ix * 8) + 4; offsets.Add(BitConverter.ToInt64(BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray <byte>()), 0)); } } } return(offsets); }
/// <summary> /// Consumes messages from Kafka. /// </summary> /// <param name="request">The request to send to Kafka.</param> /// <returns>A list of messages from Kafka.</returns> public List<Message> Consume(FetchRequest request) { List<Message> messages = new List<Message>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray<byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest byte[] unbufferedData = data.Skip(2).ToArray(); int processed = 0; int length = unbufferedData.Length - 4; int messageSize = 0; while (processed <= length) { messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray<byte>()), 0); messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray<byte>())); processed += 4 + messageSize; } } } return messages; }
/// <summary> /// Sends a request to Kafka. /// </summary> /// <param name="request">The request to send to Kafka.</param> public void Send(ProducerRequest request) { if (request.IsValid()) { using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request); } } }
private bool IsConnectionValid(KafkaConnection kafkaConnection) { bool isConnectionValid = true; isConnectionValid = lifespan > DateTime.Now.Subtract(kafkaConnection.TimeCreated); if (isConnectionValid) { isConnectionValid = kafkaConnection.IsSocketConnected(); } return(isConnectionValid); }
/// <summary> /// returns connection back to pool /// </summary> /// <param name="kafkaConnection"></param> public void ReleaseConnection(KafkaConnection kafkaConnection) { lock (availableConnections) { if (availableConnections.Count < maxPoolSize && IsConnectionValid(kafkaConnection)) { availableConnections.Enqueue(kafkaConnection); return; } System.Threading.Interlocked.Decrement(ref socketCounter); kafkaConnection.Dispose(); } }
public FetchResponse Fetch(FetchRequest request) { using (KafkaConnection connection = new KafkaConnection(server, port)) { connection.Write(request.GetRequestBytes().ToArray()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); var fetchResponse = new FetchResponse(data); return fetchResponse; } return null; } }
public FetchResponse Fetch(FetchRequest request) { using (KafkaConnection connection = new KafkaConnection(server, port)) { connection.Write(request.GetRequestBytes().ToArray()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); var fetchResponse = new FetchResponse(data); return(fetchResponse); } return(null); } }
public static void ReleaseConnection(KafkaConnection kafkaConnection) { Guard.NotNull(kafkaConnection, "kafkaConnection"); lock (locker) { string connectionKey = GetConnectionKey(kafkaConnection.Server, kafkaConnection.Port); if (kafkaServerConnections.ContainsKey(connectionKey)) { kafkaServerConnections[connectionKey].ReleaseConnection(kafkaConnection); } else { kafkaConnection.Dispose(); } } }
/// <summary> /// Get a list of valid offsets (up to maxSize) before the given time. /// </summary> /// <param name="request">The offset request.</param> /// <returns>List of offsets, in descending order.</returns> public OffsetResponse GetOffsetResponseBefore(OffsetRequest request) { using (var connection = new KafkaConnection(server, port)) { connection.Write(request.GetRequestBytes().ToArray()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength == 0) { return(null); } byte[] data = connection.Read(dataLength); var offsetResponse = new OffsetResponse(data); return(offsetResponse); } }
/// <summary> /// Send a request to Kafka asynchronously. /// </summary> /// <remarks> /// If the callback is not specified then the method behaves as a fire-and-forget call /// with the callback being ignored. By the time this callback is executed, the /// <see cref="RequestContext.NetworkStream"/> will already have been closed given an /// internal call <see cref="NetworkStream.EndWrite"/>. /// </remarks> /// <param name="request">The request to send to Kafka.</param> /// <param name="callback"> /// A block of code to execute once the request has been sent to Kafka. This value may /// be set to null. /// </param> public void SendAsync(ProducerRequest request, MessageSent <ProducerRequest> callback) { if (request.IsValid()) { KafkaConnection connection = new KafkaConnection(Server, Port); if (callback == null) { // fire and forget connection.BeginWrite(request.GetBytes()); } else { // execute with callback connection.BeginWrite(request, callback); } } }
/// <summary> /// Get meta data for a topic /// </summary> /// <param name="correlationId"></param>Id used by the client to identify this transaction. Returned in the response /// <param name="clientId"></param>Name to identify the client. Used in server logs /// <param name="topicName"></param> Name of the requested topic. If topic name is null metadata for all topics will be returned /// <returns></returns> public MetadataResponse Metadata(int correlationId, string clientId, String topicName) { MetadataRequest request = new MetadataRequest(correlationId, clientId, topicName); using (var connection = new KafkaConnection(server, port)) { connection.Write(request.GetRequestBytes().ToArray()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength == 0) { return(null); } byte[] data = connection.Read(dataLength); MetadataResponse metadataResponse = new MetadataResponse(); metadataResponse.Parse(data, 0); return(metadataResponse); } }
/// <summary> /// Returns an existing or new kafka connection /// </summary> /// <param name="server">The server to connect to.</param> /// <param name="port">The port to connect to.</param> /// <param name="bufferSize"></param> /// <param name="socketTimeout"></param> /// <param name="idleTimeToKeepAlive">idle time until keepalives (ms)</param> /// <param name="keepAliveInterval">interval between keepalives(ms)</param> /// <param name="socketPollingTimeout">socket polling timeout(usec)</param> /// <param name="keepAliveInterval">interval between keepalives(ms)</param> public KafkaConnection GetConnection() { KafkaConnection kafkaConnection = null; while (availableConnections.TryDequeue(out kafkaConnection)) { //validate if (IsConnectionValid(kafkaConnection)) { return(kafkaConnection); } else { System.Threading.Interlocked.Decrement(ref socketCounter); kafkaConnection.Dispose(); } } return(BuildConnection()); }
public ProduceResponse Produce(int correlationId, string clientId, int timeOut, string topicName, int partitionId, byte[] payLoad) { var request = new ProduceRequest(timeOut, correlationId, clientId); request.AddMessage(topicName, partitionId, payLoad); using (var connection = new KafkaConnection(server, port)) { connection.Write(request.GetRequestBytes().ToArray()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); var response = new ProduceResponse(); if (dataLength != 0) { byte[] data = connection.Read(dataLength); response.Parse(data); } return(response); } }
/// <summary> /// Get a list of valid offsets (up to maxSize) before the given time. /// </summary> /// <param name="request">The offset request.</param> /// <returns>List of offsets, in descending order.</returns> public IList <long> GetOffsetsBefore(OffsetRequest request) { List <long> offsets = new List <long>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest byte[] unbufferedData = data.Skip(2).ToArray(); // first four bytes are the number of offsets int numOfOffsets = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray <byte>()), 0); int position = 0; for (int ix = 0; ix < numOfOffsets; ix++) { position = (ix * 8) + 4; offsets.Add(BitConverter.ToInt64(BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray <byte>()), 0)); } } } return(offsets); }
/// <summary> /// Consumes messages from Kafka. /// </summary> /// <param name="request">The request to send to Kafka.</param> /// <returns>A list of messages from Kafka.</returns> public List <Message> Consume(FetchRequest request) { List <Message> messages = new List <Message>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest byte[] unbufferedData = data.Skip(2).ToArray(); int processed = 0; int length = unbufferedData.Length - 4; int messageSize = 0; while (processed <= length) { messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray <byte>()), 0); messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray <byte>())); processed += 4 + messageSize; } } } return(messages); }
private KafkaConnection BuildConnection() { KafkaConnection newConnection = null; if (socketCounter < maxPoolSize) { newConnection = new KafkaConnection(host, port, bufferSize, socketTimeout, idleTimeToKeepAlive, keepAliveInterval, socketPollingIntervalTimeout, socketPollingLevel); System.Threading.Interlocked.Increment(ref socketCounter); } else { throw new KafkaConnectionPoolException("Kafka Connection Pool reached its limit"); } return(newConnection); }
/// <summary> /// Get a list of valid offsets (up to maxSize) before the given time. /// </summary> /// <param name="request">The offset request.</param> /// <returns>List of offsets, in descending order.</returns> public OffsetResponse GetOffsetResponseBefore(OffsetRequest request) { using (var connection = new KafkaConnection(server, port)) { connection.Write(request.GetRequestBytes().ToArray()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength == 0) return null; byte[] data = connection.Read(dataLength); var offsetResponse = new OffsetResponse(data); return offsetResponse; } }
/// <summary> /// Get a list of valid offsets (up to maxSize) before the given time. /// </summary> /// <param name="request">The offset request.</param> /// <returns>List of offsets, in descending order.</returns> public IList<long> GetOffsetsBefore(OffsetRequest request) { List<long> offsets = new List<long>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray<byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest byte[] unbufferedData = data.Skip(2).ToArray(); // first four bytes are the number of offsets int numOfOffsets = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray<byte>()), 0); int position = 0; for (int ix = 0; ix < numOfOffsets; ix++) { position = (ix * 8) + 4; offsets.Add(BitConverter.ToInt64(BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray<byte>()), 0)); } } } return offsets; }
/// <summary> /// Executes a multi-fetch operation. /// </summary> /// <param name="request">The request to push to Kafka.</param> /// <returns> /// A list containing sets of messages. The message sets should match the request order. /// </returns> public List<List<Message>> Consume(MultiFetchRequest request) { int fetchRequests = request.ConsumerRequests.Count; List<List<Message>> messages = new List<List<Message>>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); int position = 0; int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray<byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest position = position + 2; for (int ix = 0; ix < fetchRequests; ix++) { messages.Add(new List<Message>()); int messageSetSize = BitConverter.ToInt32(BitWorks.ReverseBytes(data.Skip(position).Take(4).ToArray<byte>()), 0); position = position + 4; errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Skip(position).Take(2).ToArray<byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest position = position + 2; byte[] messageSetBytes = data.Skip(position).ToArray<byte>().Take(messageSetSize).ToArray<byte>(); int processed = 0; int messageSize = 0; // dropped 2 bytes at the end...padding??? while (processed < messageSetBytes.Length - 2) { messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(messageSetBytes.Skip(processed).Take(4).ToArray<byte>()), 0); messages[ix].Add(Message.ParseFrom(messageSetBytes.Skip(processed).Take(messageSize + 4).ToArray<byte>())); processed += 4 + messageSize; } position = position + processed; } } } return messages; }
public ProduceResponse Produce(int correlationId, string clientId, int timeOut, string topicName, int partitionId, byte[] payLoad) { var request = new ProduceRequest(timeOut, correlationId, clientId); request.AddMessage(topicName, partitionId, payLoad); using (var connection = new KafkaConnection(server, port)) { connection.Write(request.GetRequestBytes().ToArray()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); var response = new ProduceResponse(); if (dataLength != 0) { byte[] data = connection.Read(dataLength); response.Parse(data); } return response; } }
/// <summary> /// Get a list of valid offsets (up to maxSize) before the given time. /// </summary> /// <param name="request">The offset request.</param> /// <returns>List of offsets, in descending order.</returns> public IList<long> GetOffsetsBefore(OffsetRequest request) { List<long> offsets = new List<long>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); // TODO: need to check in on kafka error codes...assume all's good for now byte[] unbufferedData = data.Skip(2).ToArray(); // first four bytes are the number of offsets int numOfOffsets = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray<byte>()), 0); int position = 0; for (int ix = 0; ix < numOfOffsets; ix++) { position = (ix * 8) + 4; offsets.Add(BitConverter.ToInt64(BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray<byte>()), 0)); } } } return offsets; }
/// <summary> /// Consumes messages from Kafka. /// </summary> /// <param name="request">The request to send to Kafka.</param> /// <returns>A list of messages from Kafka.</returns> public List<Message> Consume(ConsumerRequest request) { List<Message> messages = new List<Message>(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); // TODO: need to check in on kafka error codes...assume all's good for now byte[] unbufferedData = data.Skip(2).ToArray(); int processed = 0; int length = unbufferedData.Length - 4; int messageSize = 0; while (processed <= length) { messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray<byte>()), 0); messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray<byte>())); processed += 4 + messageSize; } } } return messages; }
/// <summary> /// Executes a multi-fetch operation. /// </summary> /// <param name="request">The request to push to Kafka.</param> /// <returns> /// A list containing sets of messages. The message sets should match the request order. /// </returns> public List <List <Message> > Consume(MultiFetchRequest request) { int fetchRequests = request.ConsumerRequests.Count; List <List <Message> > messages = new List <List <Message> >(); using (KafkaConnection connection = new KafkaConnection(Server, Port)) { connection.Write(request.GetBytes()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength > 0) { byte[] data = connection.Read(dataLength); int position = 0; int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest position = position + 2; for (int ix = 0; ix < fetchRequests; ix++) { messages.Add(new List <Message>()); int messageSetSize = BitConverter.ToInt32(BitWorks.ReverseBytes(data.Skip(position).Take(4).ToArray <byte>()), 0); position = position + 4; errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Skip(position).Take(2).ToArray <byte>()), 0); if (errorCode != KafkaException.NoError) { throw new KafkaException(errorCode); } // skip the error code and process the rest position = position + 2; byte[] messageSetBytes = data.Skip(position).ToArray <byte>().Take(messageSetSize).ToArray <byte>(); int processed = 0; int messageSize = 0; // dropped 2 bytes at the end...padding??? while (processed < messageSetBytes.Length - 2) { messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(messageSetBytes.Skip(processed).Take(4).ToArray <byte>()), 0); messages[ix].Add(Message.ParseFrom(messageSetBytes.Skip(processed).Take(messageSize + 4).ToArray <byte>())); processed += 4 + messageSize; } position = position + processed; } } } return(messages); }
/// <summary> /// Get meta data for a topic /// </summary> /// <param name="correlationId"></param>Id used by the client to identify this transaction. Returned in the response /// <param name="clientId"></param>Name to identify the client. Used in server logs /// <param name="topicName"></param> Name of the requested topic. If topic name is null metadata for all topics will be returned /// <returns></returns> public MetadataResponse Metadata(int correlationId, string clientId, String topicName) { MetadataRequest request = new MetadataRequest(correlationId, clientId, topicName); using (var connection = new KafkaConnection(server, port)) { connection.Write(request.GetRequestBytes().ToArray()); int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0); if (dataLength == 0) return null; byte[] data = connection.Read(dataLength); MetadataResponse metadataResponse = new MetadataResponse(); metadataResponse.Parse(data, 0); return metadataResponse; } }
/// <summary> /// Send a request to Kafka asynchronously. /// </summary> /// <remarks> /// If the callback is not specified then the method behaves as a fire-and-forget call /// with the callback being ignored. By the time this callback is executed, the /// <see cref="RequestContext.NetworkStream"/> will already have been closed given an /// internal call <see cref="NetworkStream.EndWrite"/>. /// </remarks> /// <param name="request">The request to send to Kafka.</param> /// <param name="callback"> /// A block of code to execute once the request has been sent to Kafka. This value may /// be set to null. /// </param> public void SendAsync(ProducerRequest request, MessageSent<ProducerRequest> callback) { if (request.IsValid()) { KafkaConnection connection = new KafkaConnection(Server, Port); if (callback == null) { // fire and forget connection.BeginWrite(request.GetBytes()); } else { // execute with callback connection.BeginWrite(request, callback); } } }