public SocketPayloadSendTask(KafkaDataPayload payload, CancellationToken cancellationToken, IKafkaLog log)
{
_log = log;
Tcp = new TaskCompletionSource <KafkaDataPayload>();
Payload = payload;
_cancellationTokenRegistration = cancellationToken.Register(() => Tcp.TrySetCanceled());
}
private KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout, KafkaOptions kafkaOptions)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_processNetworkstreamTasksAction = ProcessNetworkstreamTasks;
_allowSelfSignedServerCert = kafkaOptions?.TslAllowSelfSignedServerCert;
if (!string.IsNullOrWhiteSpace(kafkaOptions?.TslClientCertPfxPathOrCertStoreSubject))
{
_selfSignedTrainMode = kafkaOptions.TslSelfSignedTrainMode;
_clientCert = GetClientCert(kafkaOptions.TslClientCertPfxPathOrCertStoreSubject, kafkaOptions.TslClientCertStoreFriendlyName, kafkaOptions?.TslClientCertPassword);
_processNetworkstreamTasksAction = ProcessNetworkstreamTasksTsl;
}
_sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection <SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
private static IPAddress GetFirstAddress(string hostname, IKafkaLog log)
{
try
{
//lookup the IP address from the provided host name
var addresses = Dns.GetHostAddresses(hostname);
if (addresses.Length > 0)
{
Array.ForEach(addresses, address => log.DebugFormat("Found address {0} for {1}", address, hostname));
var selectedAddress = addresses.FirstOrDefault(item => item.AddressFamily == AddressFamily.InterNetwork) ?? addresses.First();
log.DebugFormat("Using address {0} for {1}", selectedAddress, hostname);
return selectedAddress;
}
}
catch
{
throw new UnresolvedHostnameException("Could not resolve the following hostname: {0}", hostname);
}
throw new UnresolvedHostnameException("Could not resolve the following hostname: {0}", hostname);
}
private static IPAddress GetFirstAddress(string hostname, IKafkaLog log)
{
try
{
//lookup the IP address from the provided host name
var addresses = Dns.GetHostAddresses(hostname);
if (addresses.Length > 0)
{
Array.ForEach(addresses, address => log.DebugFormat("Found address {0} for {1}", address, hostname));
var selectedAddress = addresses.FirstOrDefault(item => item.AddressFamily == AddressFamily.InterNetwork) ?? addresses.First();
log.DebugFormat("Using address {0} for {1}", selectedAddress, hostname);
return(selectedAddress);
}
}
catch
{
throw new UnresolvedHostnameException("Could not resolve the following hostname: {0}", hostname);
}
throw new UnresolvedHostnameException("Could not resolve the following hostname: {0}", hostname);
}
public SocketPayloadReadTask(int readSize, CancellationToken cancellationToken, IKafkaLog log)
{
_log = log;
CancellationToken = cancellationToken;
Tcp = new TaskCompletionSource <byte[]>();
ReadSize = readSize;
_cancellationTokenRegistration = cancellationToken.Register(() => Tcp.TrySetCanceled());
}
public KafkaOptions(params Uri[] kafkaServerUri)
{
KafkaServerUri = kafkaServerUri.ToList();
PartitionSelector = new DefaultPartitionSelector();
Log = new DefaultTraceLog();
KafkaConnectionFactory = new DefaultKafkaConnectionFactory();
ResponseTimeoutMs = TimeSpan.FromMilliseconds(DefaultResponseTimeout);
}
/// <summary>
/// Initializes a new instance of the KafkaConnection class.
/// </summary>
/// <param name="log">Logging interface used to record any log messages created by the connection.</param>
/// <param name="client">The kafka socket initialized to the kafka server.</param>
/// <param name="responseTimeoutMs">The amount of time to wait for a message response to be received after sending message to Kafka. Defaults to 30s.</param>
public KafkaConnection(IKafkaTcpSocket client, TimeSpan?responseTimeoutMs = null, IKafkaLog log = null)
{
_client = client;
_log = log ?? new DefaultTraceLog();
_responseTimeoutMs = responseTimeoutMs ?? TimeSpan.FromMilliseconds(DefaultResponseTimeoutMs);
StartReadStreamPoller();
}
public KafkaOptions(params Uri[] kafkaServerUri)
{
KafkaServerUri = kafkaServerUri.ToList();
PartitionSelector = new DefaultPartitionSelector();
Log = new DefaultTraceLog();
KafkaConnectionFactory = new DefaultKafkaConnectionFactory();
ResponseTimeoutMs = TimeSpan.FromMilliseconds(DefaultResponseTimeout);
}
/// <summary>
/// Initializes a new instance of the KafkaConnection class.
/// </summary>
/// <param name="log">Logging interface used to record any log messages created by the connection.</param>
/// <param name="serverAddress">The Uri address to this kafka server.</param>
/// <param name="responseTimeoutMs">The amount of time to wait for a message response to be received from kafka.</param>
public KafkaConnection(Uri serverAddress, int responseTimeoutMs = DefaultResponseTimeoutMs, IKafkaLog log = null)
{
_log = log ?? new DefaultTraceLog();
_kafkaUri = serverAddress;
_responseTimeoutMS = responseTimeoutMs;
_responseTimeoutTimer = new ScheduledTimer()
.Do(ResponseTimeoutCheck)
.Every(TimeSpan.FromMilliseconds(100))
.StartingAt(DateTime.Now.AddMilliseconds(_responseTimeoutMS))
.Begin();
}
public KafkaEndpoint Resolve(Uri kafkaAddress, IKafkaLog log)
{
var ipAddress = GetFirstAddress(kafkaAddress.Host, log);
var ipEndpoint = new IPEndPoint(ipAddress, kafkaAddress.Port);
var kafkaEndpoint = new KafkaEndpoint()
{
ServeUri = kafkaAddress,
Endpoint = ipEndpoint
};
return(kafkaEndpoint);
}
/// <summary>
/// Initializes a new instance of the KafkaConnection class.
/// </summary>
/// <param name="log">Logging interface used to record any log messages created by the connection.</param>
/// <param name="client">The kafka socket initialized to the kafka server.</param>
/// <param name="responseTimeoutMs">The amount of time to wait for a message response to be received after sending message to Kafka. Defaults to 30s.</param>
public KafkaConnection(IKafkaTcpSocket client, TimeSpan?responseTimeoutMs = null, IKafkaLog log = null)
{
_client = client;
_log = log ?? new DefaultTraceLog();
_responseTimeoutMS = responseTimeoutMs ?? TimeSpan.FromMilliseconds(DefaultResponseTimeoutMs);
_responseTimeoutTimer = new ScheduledTimer()
.Do(ResponseTimeoutCheck)
.Every(TimeSpan.FromMilliseconds(100))
.StartingAt(DateTime.Now.Add(_responseTimeoutMS))
.Begin();
StartReadStreamPoller();
}
public KafkaEndpoint Resolve(Uri kafkaAddress, IKafkaLog log)
{
var ipAddress = GetFirstAddress(kafkaAddress.Host, log);
var ipEndpoint = new IPEndPoint(ipAddress, kafkaAddress.Port);
var kafkaEndpoint = new KafkaEndpoint()
{
ServeUri = kafkaAddress,
Endpoint = ipEndpoint
};
return kafkaEndpoint;
}
/// <summary>
/// Initializes a new instance of the KafkaConnection class.
/// </summary>
/// <param name="log">Logging interface used to record any log messages created by the connection.</param>
/// <param name="serverAddress">The Uri address to this kafka server.</param>
/// <param name="responseTimeoutMs">The amount of time to wait for a message response to be received from kafka.</param>
public KafkaConnection(Uri serverAddress, int responseTimeoutMs = DefaultResponseTimeoutMs, IKafkaLog log = null)
{
_readerSemaphore = new SemaphoreSlim(1, 1);
_responseTimeoutSemaphore = new SemaphoreSlim(1, 1);
_log = log ?? new DefaultTraceLog();
_kafkaUri = serverAddress;
_responseTimeoutMS = responseTimeoutMs;
_cancellationTokenSource = new CancellationTokenSource();
_responseTimeoutTimer = new ScheduledTimer()
.Do(ResponseTimeoutCheck)
.Every(TimeSpan.FromMilliseconds(100))
.StartingAt(DateTime.Now.AddMilliseconds(_responseTimeoutMS))
.Begin();
}
public static void DisposeSafely(this object objToDispose, IKafkaLog log)
{
var disposable = objToDispose as IDisposable;
if (disposable != null)
{
var className = disposable.GetType().Name;
try
{
disposable.Dispose();
log?.DebugFormat("Successfully disposed {0}", className);
}
catch (Exception e)
{
log?.WarnFormat("Error disposing {0}, Exception {1}", className, e);
}
}
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection<SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Run(async () => { await DedicatedSocketTask(); });
_disposeTask = _disposeToken.Token.CreateTask();
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection <SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection<SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
public FakeTcpServer(IKafkaLog log, int port)
{
_log = log;
_listener = new TcpListener(IPAddress.Any, port);
_listener.Start();
OnClientConnected += () =>
{
Interlocked.Increment(ref ConnectionEventcount);
_clientConnectedTrigger.TrySetResult(true);
};
OnClientDisconnected += () =>
{
Interlocked.Increment(ref DisconnectionEventCount);
_clientConnectedTrigger = new TaskCompletionSource<bool>();
};
_clientConnectionHandlerTask = StartHandlingClientRequestAsync();
}
public FakeTcpServer(IKafkaLog log, int port)
{
_log = log;
_listener = new TcpListener(IPAddress.Any, port);
_listener.Start();
OnClientConnected += () =>
{
Interlocked.Increment(ref ConnectionEventcount);
_clientConnectedTrigger.TrySetResult(true);
};
OnClientDisconnected += () =>
{
Interlocked.Increment(ref DisconnectionEventCount);
_clientConnectedTrigger = new TaskCompletionSource <bool>();
};
_clientConnectionHandlerTask = StartHandlingClientRequestAsync();
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, int maxRetry, TimeSpan?maximumReconnectionTimeout = null, StatisticsTrackerOptions statisticsTrackerOptions = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_maxRetry = maxRetry;
_statisticsTrackerOptions = statisticsTrackerOptions;
_sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection <SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Run(async() => { await DedicatedSocketTask(); });
_disposeTask = _disposeToken.Token.CreateTask();
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
public KafkaMetadataProvider(IKafkaLog log)
{
_log = log;
}
public void Setup()
{
_log = Substitute.For <IKafkaLog>();
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="delayConnectAttemptMS">Time in milliseconds to delay the initial connection attempt to the given server.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, int delayConnectAttemptMS = 0)
{
_log = log;
_endpoint = endpoint;
Task.Delay(TimeSpan.FromMilliseconds(delayConnectAttemptMS)).ContinueWith(x => TriggerReconnection());
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint)
{
_log = log;
_endpoint = endpoint;
}
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log)
{
return(new KafkaConnection(new KafkaTcpSocket(log, endpoint), responseTimeoutMs, log));
}
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, TimeSpan? maximumReconnectionTimeout = null)
{
return new KafkaConnection(new KafkaTcpSocket(log, endpoint, maximumReconnectionTimeout), responseTimeoutMs, log);
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null) : this(log, endpoint, maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes), null)
{
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="serverUri">The server to connect to.</param>
/// <param name="delayConnectAttemptMS">Time in milliseconds to delay the initial connection attempt to the given server.</param>
public KafkaTcpSocket(IKafkaLog log, Uri serverUri, int delayConnectAttemptMS = 0)
{
_log = log;
_serverUri = serverUri;
Task.Delay(TimeSpan.FromMilliseconds(delayConnectAttemptMS)).ContinueWith(x => TriggerReconnection());
}
public IKafkaConnection Create(Uri kafkaAddress, int responseTimeoutMs, IKafkaLog log)
{
return(new KafkaConnection(new KafkaTcpSocket(log, kafkaAddress), responseTimeoutMs, log));
}
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, int maxRetry, TimeSpan?maximumReconnectionTimeout = null, StatisticsTrackerOptions statisticsTrackerOptions = null)
{
KafkaTcpSocket socket = new KafkaTcpSocket(log, endpoint, maxRetry, maximumReconnectionTimeout, statisticsTrackerOptions);
return(new KafkaConnection(socket, responseTimeoutMs, log));
}
public IKafkaConnection Create(Uri kafkaAddress, int responseTimeoutMs, IKafkaLog log)
{
return new KafkaConnection(kafkaAddress, responseTimeoutMs, log);
}
public KafkaMetadataProvider(IKafkaLog log)
{
_log = log;
}
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, TimeSpan?maximumReconnectionTimeout = null)
{
return(new KafkaConnection(new KafkaTcpSocket(log, endpoint, maximumReconnectionTimeout), responseTimeoutMs, log));
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="delayConnectAttemptMS">Time in milliseconds to delay the initial connection attempt to the given server.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, int delayConnectAttemptMS = 0)
{
_log = log;
_endpoint = endpoint;
Task.Delay(TimeSpan.FromMilliseconds(delayConnectAttemptMS)).ContinueWith(x => TriggerReconnection());
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
}
public void Setup()
{
_log = Substitute.For<IKafkaLog>();
}
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log)
{
return new KafkaConnection(new KafkaTcpSocket(log, endpoint), responseTimeoutMs, log);
}
