private KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout, KafkaOptions kafkaOptions)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_processNetworkstreamTasksAction = ProcessNetworkstreamTasks;
_allowSelfSignedServerCert = kafkaOptions?.TslAllowSelfSignedServerCert;
if (!string.IsNullOrWhiteSpace(kafkaOptions?.TslClientCertPfxPathOrCertStoreSubject))
{
_selfSignedTrainMode = kafkaOptions.TslSelfSignedTrainMode;
_clientCert = GetClientCert(kafkaOptions.TslClientCertPfxPathOrCertStoreSubject, kafkaOptions.TslClientCertStoreFriendlyName, kafkaOptions?.TslClientCertPassword);
_processNetworkstreamTasksAction = ProcessNetworkstreamTasksTsl;
}
_sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection <SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
public KafkaTcpSocketTests()
{
var log = new DefaultTraceLog();
_fakeServerUrl = new DefaultKafkaConnectionFactory().Resolve(new Uri("http://localhost:8999"), log);
_badServerUrl = new DefaultKafkaConnectionFactory().Resolve(new Uri("http://localhost:1"), log);
}
public BrokerException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
bool hasFetch = info.GetByte("HasBrokerEndPoint") == 1;
if (hasFetch)
{
IPEndPoint ipEndPoint = null;
Uri uri = null;
if (info.GetByte("HasEndpoint") == 1)
{
ipEndPoint = new IPEndPoint(info.GetInt64("Address"), info.GetInt32("Port"));
}
if (info.GetByte("HasServeUri") == 1)
{
uri = new Uri(info.GetString("ServeUri"));
}
BrokerEndPoint = new KafkaEndpoint
{
Endpoint = ipEndPoint,
ServeUri = uri
};
}
}
public ConfluentKafkaListener(KafkaEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation)
{
_endpoint = endpoint;
_logger = logger;
_cancellation = cancellation;
_protocol = new KafkaTransportProtocol();
_consumer = new ConsumerBuilder <byte[], byte[]>(endpoint.ConsumerConfig).Build();
}
public void parse_non_durable_uri()
{
var endpoint = new KafkaEndpoint();
endpoint.Parse(new Uri("kafka://topic/key1"));
endpoint.Mode.ShouldBe(EndpointMode.BufferedInMemory);
endpoint.TopicName.ShouldBe("key1");
}
public void parse_durable_uri()
{
var endpoint = new KafkaEndpoint();
endpoint.Parse(new Uri("kafka://topic/key1/durable"));
endpoint.Mode.ShouldBe(EndpointMode.Durable);
endpoint.TopicName.ShouldBe("key1");
}
public override Uri BuildUriForTopic(string topicName)
{
// TODO -- this probably shouldn't be durable by default, but
// that's how it was coded before
var endpoint = new KafkaEndpoint
{
Mode = EndpointMode.Durable,
TopicName = topicName
};
return(endpoint.Uri);
}
public static void QueueNetworkWrite(KafkaEndpoint endpoint, KafkaDataPayload payload)
{
if (payload.TrackPayload == false)
{
return;
}
var stat = new NetworkWriteStatistic(endpoint, payload);
NetworkWriteQueuedIndex.TryAdd(payload.CorrelationId, stat);
Interlocked.Increment(ref Gauges.QueuedWriteOperation);
}
public KafkaEndpoint Resolve(Uri kafkaAddress, IKafkaLog log)
{
var ipAddress = GetFirstAddress(kafkaAddress.Host, log);
var ipEndpoint = new IPEndPoint(ipAddress, kafkaAddress.Port);
var kafkaEndpoint = new KafkaEndpoint()
{
ServeUri = kafkaAddress,
Endpoint = ipEndpoint
};
return(kafkaEndpoint);
}
private void UpsertConnectionToBrokerConnectionIndex(int brokerId, KafkaEndpoint brokerEndpoint, Func <int, IKafkaConnection> connectionFactory)
{
//associate the connection with the broker id, and add or update the reference
_brokerConnectionIndex.AddOrUpdate(brokerId, connectionFactory,
(i, existingConnection) =>
{
//if a connection changes for a broker close old connection and create a new one
if (existingConnection.Endpoint.Equals(brokerEndpoint))
{
return(existingConnection);
}
_kafkaOptions.Log.WarnFormat("Broker:{0} Uri changed from:{1} to {2}", brokerId, existingConnection.Endpoint, brokerEndpoint);
existingConnection.Dispose();
return(connectionFactory(i));
});
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection <SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
public ConfluentKafkaSender(KafkaEndpoint endpoint)
{
if (endpoint?.ProducerConfig == null)
{
throw new ArgumentNullException(nameof(KafkaEndpoint.ProducerConfig));
}
_endpoint = endpoint;
_publisher = new ProducerBuilder <byte[], byte[]>(endpoint.ProducerConfig)
.SetErrorHandler((producer, error) =>
{
if (error.IsFatal)
{
throw new KafkaSenderException(error);
}
})
.Build();
_protocol = new KafkaTransportProtocol();
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, int maxRetry, TimeSpan?maximumReconnectionTimeout = null, StatisticsTrackerOptions statisticsTrackerOptions = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_maxRetry = maxRetry;
_statisticsTrackerOptions = statisticsTrackerOptions;
_sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection <SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Run(async() => { await DedicatedSocketTask(); });
_disposeTask = _disposeToken.Token.CreateTask();
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null) : this(log, endpoint, maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes), null)
{
}
public IKafkaConnection Create(KafkaEndpoint endpoint, KafkaOptions kafkaOptions)
{
return(new KafkaConnection(new KafkaTcpSocket(endpoint, kafkaOptions), kafkaOptions.ResponseTimeoutMs, kafkaOptions.Log));
}
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, TimeSpan?maximumReconnectionTimeout = null)
{
return(new KafkaConnection(new KafkaTcpSocket(log, endpoint, maximumReconnectionTimeout), responseTimeoutMs, log));
}
public BrokerConnectionException(string message, KafkaEndpoint endPoint, Exception innerException)
: base(message, endPoint, innerException)
{
BrokerEndPoint = endPoint;
}
public BrokerConnectionException(string message, KafkaEndpoint endPoint, params object[] args)
: base(string.Format(message, args), endPoint)
{
}
public KafkaTcpSocket(KafkaEndpoint endpoint, KafkaOptions kafkaOptions) : this(kafkaOptions.Log, endpoint, kafkaOptions.MaximumReconnectionTimeout, kafkaOptions)
{
}
public BrokerException(string message, KafkaEndpoint endPoint, params object[] args)
: base(string.Format(message, args))
{
BrokerEndPoint = endPoint;
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint)
{
_log = log;
_endpoint = endpoint;
}
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log)
{
return(new KafkaConnection(new KafkaTcpSocket(log, endpoint), responseTimeoutMs, log));
}
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, int maxRetry, TimeSpan?maximumReconnectionTimeout = null, StatisticsTrackerOptions statisticsTrackerOptions = null)
{
KafkaTcpSocket socket = new KafkaTcpSocket(log, endpoint, maxRetry, maximumReconnectionTimeout, statisticsTrackerOptions);
return(new KafkaConnection(socket, responseTimeoutMs, log));
}
public NetworkWriteStatistic(KafkaEndpoint endpoint, KafkaDataPayload payload)
{
CreatedOnUtc = DateTime.UtcNow;
Endpoint = endpoint;
Payload = payload;
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
}
public KafkaConnectionTests()
{
_log = new DefaultTraceLog();
_kafkaEndpoint = new DefaultKafkaConnectionFactory().Resolve(new Uri("http://localhost:8999"), _log);
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="delayConnectAttemptMS">Time in milliseconds to delay the initial connection attempt to the given server.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, int delayConnectAttemptMS = 0)
{
_log = log;
_endpoint = endpoint;
Task.Delay(TimeSpan.FromMilliseconds(delayConnectAttemptMS)).ContinueWith(x => TriggerReconnection());
}
