public SocketPayloadSendTask(KafkaDataPayload payload, CancellationToken cancellationToken, IKafkaLog log) { _log = log; Tcp = new TaskCompletionSource <KafkaDataPayload>(); Payload = payload; _cancellationTokenRegistration = cancellationToken.Register(() => Tcp.TrySetCanceled()); }
private KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout, KafkaOptions kafkaOptions) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); _processNetworkstreamTasksAction = ProcessNetworkstreamTasks; _allowSelfSignedServerCert = kafkaOptions?.TslAllowSelfSignedServerCert; if (!string.IsNullOrWhiteSpace(kafkaOptions?.TslClientCertPfxPathOrCertStoreSubject)) { _selfSignedTrainMode = kafkaOptions.TslSelfSignedTrainMode; _clientCert = GetClientCert(kafkaOptions.TslClientCertPfxPathOrCertStoreSubject, kafkaOptions.TslClientCertStoreFriendlyName, kafkaOptions?.TslClientCertPassword); _processNetworkstreamTasksAction = ProcessNetworkstreamTasksTsl; } _sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>(); _readTaskQueue = new AsyncCollection <SocketPayloadReadTask>(); //dedicate a long running task to the read/write operations _socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); _disposeRegistration = _disposeToken.Token.Register(() => { _sendTaskQueue.CompleteAdding(); _readTaskQueue.CompleteAdding(); }); }
private static IPAddress GetFirstAddress(string hostname, IKafkaLog log) { try { //lookup the IP address from the provided host name var addresses = Dns.GetHostAddresses(hostname); if (addresses.Length > 0) { Array.ForEach(addresses, address => log.DebugFormat("Found address {0} for {1}", address, hostname)); var selectedAddress = addresses.FirstOrDefault(item => item.AddressFamily == AddressFamily.InterNetwork) ?? addresses.First(); log.DebugFormat("Using address {0} for {1}", selectedAddress, hostname); return selectedAddress; } } catch { throw new UnresolvedHostnameException("Could not resolve the following hostname: {0}", hostname); } throw new UnresolvedHostnameException("Could not resolve the following hostname: {0}", hostname); }
private static IPAddress GetFirstAddress(string hostname, IKafkaLog log) { try { //lookup the IP address from the provided host name var addresses = Dns.GetHostAddresses(hostname); if (addresses.Length > 0) { Array.ForEach(addresses, address => log.DebugFormat("Found address {0} for {1}", address, hostname)); var selectedAddress = addresses.FirstOrDefault(item => item.AddressFamily == AddressFamily.InterNetwork) ?? addresses.First(); log.DebugFormat("Using address {0} for {1}", selectedAddress, hostname); return(selectedAddress); } } catch { throw new UnresolvedHostnameException("Could not resolve the following hostname: {0}", hostname); } throw new UnresolvedHostnameException("Could not resolve the following hostname: {0}", hostname); }
public SocketPayloadReadTask(int readSize, CancellationToken cancellationToken, IKafkaLog log) { _log = log; CancellationToken = cancellationToken; Tcp = new TaskCompletionSource <byte[]>(); ReadSize = readSize; _cancellationTokenRegistration = cancellationToken.Register(() => Tcp.TrySetCanceled()); }
public KafkaOptions(params Uri[] kafkaServerUri) { KafkaServerUri = kafkaServerUri.ToList(); PartitionSelector = new DefaultPartitionSelector(); Log = new DefaultTraceLog(); KafkaConnectionFactory = new DefaultKafkaConnectionFactory(); ResponseTimeoutMs = TimeSpan.FromMilliseconds(DefaultResponseTimeout); }
/// <summary> /// Initializes a new instance of the KafkaConnection class. /// </summary> /// <param name="log">Logging interface used to record any log messages created by the connection.</param> /// <param name="client">The kafka socket initialized to the kafka server.</param> /// <param name="responseTimeoutMs">The amount of time to wait for a message response to be received after sending message to Kafka. Defaults to 30s.</param> public KafkaConnection(IKafkaTcpSocket client, TimeSpan?responseTimeoutMs = null, IKafkaLog log = null) { _client = client; _log = log ?? new DefaultTraceLog(); _responseTimeoutMs = responseTimeoutMs ?? TimeSpan.FromMilliseconds(DefaultResponseTimeoutMs); StartReadStreamPoller(); }
/// <summary> /// Initializes a new instance of the KafkaConnection class. /// </summary> /// <param name="log">Logging interface used to record any log messages created by the connection.</param> /// <param name="serverAddress">The Uri address to this kafka server.</param> /// <param name="responseTimeoutMs">The amount of time to wait for a message response to be received from kafka.</param> public KafkaConnection(Uri serverAddress, int responseTimeoutMs = DefaultResponseTimeoutMs, IKafkaLog log = null) { _log = log ?? new DefaultTraceLog(); _kafkaUri = serverAddress; _responseTimeoutMS = responseTimeoutMs; _responseTimeoutTimer = new ScheduledTimer() .Do(ResponseTimeoutCheck) .Every(TimeSpan.FromMilliseconds(100)) .StartingAt(DateTime.Now.AddMilliseconds(_responseTimeoutMS)) .Begin(); }
public KafkaEndpoint Resolve(Uri kafkaAddress, IKafkaLog log) { var ipAddress = GetFirstAddress(kafkaAddress.Host, log); var ipEndpoint = new IPEndPoint(ipAddress, kafkaAddress.Port); var kafkaEndpoint = new KafkaEndpoint() { ServeUri = kafkaAddress, Endpoint = ipEndpoint }; return(kafkaEndpoint); }
/// <summary> /// Initializes a new instance of the KafkaConnection class. /// </summary> /// <param name="log">Logging interface used to record any log messages created by the connection.</param> /// <param name="client">The kafka socket initialized to the kafka server.</param> /// <param name="responseTimeoutMs">The amount of time to wait for a message response to be received after sending message to Kafka. Defaults to 30s.</param> public KafkaConnection(IKafkaTcpSocket client, TimeSpan?responseTimeoutMs = null, IKafkaLog log = null) { _client = client; _log = log ?? new DefaultTraceLog(); _responseTimeoutMS = responseTimeoutMs ?? TimeSpan.FromMilliseconds(DefaultResponseTimeoutMs); _responseTimeoutTimer = new ScheduledTimer() .Do(ResponseTimeoutCheck) .Every(TimeSpan.FromMilliseconds(100)) .StartingAt(DateTime.Now.Add(_responseTimeoutMS)) .Begin(); StartReadStreamPoller(); }
public KafkaEndpoint Resolve(Uri kafkaAddress, IKafkaLog log) { var ipAddress = GetFirstAddress(kafkaAddress.Host, log); var ipEndpoint = new IPEndPoint(ipAddress, kafkaAddress.Port); var kafkaEndpoint = new KafkaEndpoint() { ServeUri = kafkaAddress, Endpoint = ipEndpoint }; return kafkaEndpoint; }
/// <summary> /// Initializes a new instance of the KafkaConnection class. /// </summary> /// <param name="log">Logging interface used to record any log messages created by the connection.</param> /// <param name="serverAddress">The Uri address to this kafka server.</param> /// <param name="responseTimeoutMs">The amount of time to wait for a message response to be received from kafka.</param> public KafkaConnection(Uri serverAddress, int responseTimeoutMs = DefaultResponseTimeoutMs, IKafkaLog log = null) { _readerSemaphore = new SemaphoreSlim(1, 1); _responseTimeoutSemaphore = new SemaphoreSlim(1, 1); _log = log ?? new DefaultTraceLog(); _kafkaUri = serverAddress; _responseTimeoutMS = responseTimeoutMs; _cancellationTokenSource = new CancellationTokenSource(); _responseTimeoutTimer = new ScheduledTimer() .Do(ResponseTimeoutCheck) .Every(TimeSpan.FromMilliseconds(100)) .StartingAt(DateTime.Now.AddMilliseconds(_responseTimeoutMS)) .Begin(); }
public static void DisposeSafely(this object objToDispose, IKafkaLog log) { var disposable = objToDispose as IDisposable; if (disposable != null) { var className = disposable.GetType().Name; try { disposable.Dispose(); log?.DebugFormat("Successfully disposed {0}", className); } catch (Exception e) { log?.WarnFormat("Error disposing {0}, Exception {1}", className, e); } } }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); _sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>(); _readTaskQueue = new AsyncCollection<SocketPayloadReadTask>(); //dedicate a long running task to the read/write operations _socketTask = Task.Run(async () => { await DedicatedSocketTask(); }); _disposeTask = _disposeToken.Token.CreateTask(); _disposeRegistration = _disposeToken.Token.Register(() => { _sendTaskQueue.CompleteAdding(); _readTaskQueue.CompleteAdding(); }); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); _sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>(); _readTaskQueue = new AsyncCollection <SocketPayloadReadTask>(); //dedicate a long running task to the read/write operations _socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); _disposeRegistration = _disposeToken.Token.Register(() => { _sendTaskQueue.CompleteAdding(); _readTaskQueue.CompleteAdding(); }); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); _sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>(); _readTaskQueue = new AsyncCollection<SocketPayloadReadTask>(); //dedicate a long running task to the read/write operations _socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); _disposeRegistration = _disposeToken.Token.Register(() => { _sendTaskQueue.CompleteAdding(); _readTaskQueue.CompleteAdding(); }); }
public FakeTcpServer(IKafkaLog log, int port) { _log = log; _listener = new TcpListener(IPAddress.Any, port); _listener.Start(); OnClientConnected += () => { Interlocked.Increment(ref ConnectionEventcount); _clientConnectedTrigger.TrySetResult(true); }; OnClientDisconnected += () => { Interlocked.Increment(ref DisconnectionEventCount); _clientConnectedTrigger = new TaskCompletionSource<bool>(); }; _clientConnectionHandlerTask = StartHandlingClientRequestAsync(); }
public FakeTcpServer(IKafkaLog log, int port) { _log = log; _listener = new TcpListener(IPAddress.Any, port); _listener.Start(); OnClientConnected += () => { Interlocked.Increment(ref ConnectionEventcount); _clientConnectedTrigger.TrySetResult(true); }; OnClientDisconnected += () => { Interlocked.Increment(ref DisconnectionEventCount); _clientConnectedTrigger = new TaskCompletionSource <bool>(); }; _clientConnectionHandlerTask = StartHandlingClientRequestAsync(); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, int maxRetry, TimeSpan?maximumReconnectionTimeout = null, StatisticsTrackerOptions statisticsTrackerOptions = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); _maxRetry = maxRetry; _statisticsTrackerOptions = statisticsTrackerOptions; _sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>(); _readTaskQueue = new AsyncCollection <SocketPayloadReadTask>(); //dedicate a long running task to the read/write operations _socketTask = Task.Run(async() => { await DedicatedSocketTask(); }); _disposeTask = _disposeToken.Token.CreateTask(); _disposeRegistration = _disposeToken.Token.Register(() => { _sendTaskQueue.CompleteAdding(); _readTaskQueue.CompleteAdding(); }); }
public KafkaMetadataProvider(IKafkaLog log) { _log = log; }
public void Setup() { _log = Substitute.For <IKafkaLog>(); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="delayConnectAttemptMS">Time in milliseconds to delay the initial connection attempt to the given server.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, int delayConnectAttemptMS = 0) { _log = log; _endpoint = endpoint; Task.Delay(TimeSpan.FromMilliseconds(delayConnectAttemptMS)).ContinueWith(x => TriggerReconnection()); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint) { _log = log; _endpoint = endpoint; }
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log) { return(new KafkaConnection(new KafkaTcpSocket(log, endpoint), responseTimeoutMs, log)); }
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, TimeSpan? maximumReconnectionTimeout = null) { return new KafkaConnection(new KafkaTcpSocket(log, endpoint, maximumReconnectionTimeout), responseTimeoutMs, log); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null) : this(log, endpoint, maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes), null) { }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="serverUri">The server to connect to.</param> /// <param name="delayConnectAttemptMS">Time in milliseconds to delay the initial connection attempt to the given server.</param> public KafkaTcpSocket(IKafkaLog log, Uri serverUri, int delayConnectAttemptMS = 0) { _log = log; _serverUri = serverUri; Task.Delay(TimeSpan.FromMilliseconds(delayConnectAttemptMS)).ContinueWith(x => TriggerReconnection()); }
public IKafkaConnection Create(Uri kafkaAddress, int responseTimeoutMs, IKafkaLog log) { return(new KafkaConnection(new KafkaTcpSocket(log, kafkaAddress), responseTimeoutMs, log)); }
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, int maxRetry, TimeSpan?maximumReconnectionTimeout = null, StatisticsTrackerOptions statisticsTrackerOptions = null) { KafkaTcpSocket socket = new KafkaTcpSocket(log, endpoint, maxRetry, maximumReconnectionTimeout, statisticsTrackerOptions); return(new KafkaConnection(socket, responseTimeoutMs, log)); }
public IKafkaConnection Create(Uri kafkaAddress, int responseTimeoutMs, IKafkaLog log) { return new KafkaConnection(kafkaAddress, responseTimeoutMs, log); }
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, TimeSpan?maximumReconnectionTimeout = null) { return(new KafkaConnection(new KafkaTcpSocket(log, endpoint, maximumReconnectionTimeout), responseTimeoutMs, log)); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); }
public void Setup() { _log = Substitute.For<IKafkaLog>(); }
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log) { return new KafkaConnection(new KafkaTcpSocket(log, endpoint), responseTimeoutMs, log); }