private KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout, KafkaOptions kafkaOptions) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); _processNetworkstreamTasksAction = ProcessNetworkstreamTasks; _allowSelfSignedServerCert = kafkaOptions?.TslAllowSelfSignedServerCert; if (!string.IsNullOrWhiteSpace(kafkaOptions?.TslClientCertPfxPathOrCertStoreSubject)) { _selfSignedTrainMode = kafkaOptions.TslSelfSignedTrainMode; _clientCert = GetClientCert(kafkaOptions.TslClientCertPfxPathOrCertStoreSubject, kafkaOptions.TslClientCertStoreFriendlyName, kafkaOptions?.TslClientCertPassword); _processNetworkstreamTasksAction = ProcessNetworkstreamTasksTsl; } _sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>(); _readTaskQueue = new AsyncCollection <SocketPayloadReadTask>(); //dedicate a long running task to the read/write operations _socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); _disposeRegistration = _disposeToken.Token.Register(() => { _sendTaskQueue.CompleteAdding(); _readTaskQueue.CompleteAdding(); }); }
public KafkaTcpSocketTests() { var log = new DefaultTraceLog(); _fakeServerUrl = new DefaultKafkaConnectionFactory().Resolve(new Uri("http://localhost:8999"), log); _badServerUrl = new DefaultKafkaConnectionFactory().Resolve(new Uri("http://localhost:1"), log); }
public BrokerException(SerializationInfo info, StreamingContext context) : base(info, context) { bool hasFetch = info.GetByte("HasBrokerEndPoint") == 1; if (hasFetch) { IPEndPoint ipEndPoint = null; Uri uri = null; if (info.GetByte("HasEndpoint") == 1) { ipEndPoint = new IPEndPoint(info.GetInt64("Address"), info.GetInt32("Port")); } if (info.GetByte("HasServeUri") == 1) { uri = new Uri(info.GetString("ServeUri")); } BrokerEndPoint = new KafkaEndpoint { Endpoint = ipEndPoint, ServeUri = uri }; } }
public ConfluentKafkaListener(KafkaEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation) { _endpoint = endpoint; _logger = logger; _cancellation = cancellation; _protocol = new KafkaTransportProtocol(); _consumer = new ConsumerBuilder <byte[], byte[]>(endpoint.ConsumerConfig).Build(); }
public void parse_non_durable_uri() { var endpoint = new KafkaEndpoint(); endpoint.Parse(new Uri("kafka://topic/key1")); endpoint.Mode.ShouldBe(EndpointMode.BufferedInMemory); endpoint.TopicName.ShouldBe("key1"); }
public void parse_durable_uri() { var endpoint = new KafkaEndpoint(); endpoint.Parse(new Uri("kafka://topic/key1/durable")); endpoint.Mode.ShouldBe(EndpointMode.Durable); endpoint.TopicName.ShouldBe("key1"); }
public override Uri BuildUriForTopic(string topicName) { // TODO -- this probably shouldn't be durable by default, but // that's how it was coded before var endpoint = new KafkaEndpoint { Mode = EndpointMode.Durable, TopicName = topicName }; return(endpoint.Uri); }
public static void QueueNetworkWrite(KafkaEndpoint endpoint, KafkaDataPayload payload) { if (payload.TrackPayload == false) { return; } var stat = new NetworkWriteStatistic(endpoint, payload); NetworkWriteQueuedIndex.TryAdd(payload.CorrelationId, stat); Interlocked.Increment(ref Gauges.QueuedWriteOperation); }
public KafkaEndpoint Resolve(Uri kafkaAddress, IKafkaLog log) { var ipAddress = GetFirstAddress(kafkaAddress.Host, log); var ipEndpoint = new IPEndPoint(ipAddress, kafkaAddress.Port); var kafkaEndpoint = new KafkaEndpoint() { ServeUri = kafkaAddress, Endpoint = ipEndpoint }; return(kafkaEndpoint); }
private void UpsertConnectionToBrokerConnectionIndex(int brokerId, KafkaEndpoint brokerEndpoint, Func <int, IKafkaConnection> connectionFactory) { //associate the connection with the broker id, and add or update the reference _brokerConnectionIndex.AddOrUpdate(brokerId, connectionFactory, (i, existingConnection) => { //if a connection changes for a broker close old connection and create a new one if (existingConnection.Endpoint.Equals(brokerEndpoint)) { return(existingConnection); } _kafkaOptions.Log.WarnFormat("Broker:{0} Uri changed from:{1} to {2}", brokerId, existingConnection.Endpoint, brokerEndpoint); existingConnection.Dispose(); return(connectionFactory(i)); }); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); _sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>(); _readTaskQueue = new AsyncCollection <SocketPayloadReadTask>(); //dedicate a long running task to the read/write operations _socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); _disposeRegistration = _disposeToken.Token.Register(() => { _sendTaskQueue.CompleteAdding(); _readTaskQueue.CompleteAdding(); }); }
public ConfluentKafkaSender(KafkaEndpoint endpoint) { if (endpoint?.ProducerConfig == null) { throw new ArgumentNullException(nameof(KafkaEndpoint.ProducerConfig)); } _endpoint = endpoint; _publisher = new ProducerBuilder <byte[], byte[]>(endpoint.ProducerConfig) .SetErrorHandler((producer, error) => { if (error.IsFatal) { throw new KafkaSenderException(error); } }) .Build(); _protocol = new KafkaTransportProtocol(); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, int maxRetry, TimeSpan?maximumReconnectionTimeout = null, StatisticsTrackerOptions statisticsTrackerOptions = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); _maxRetry = maxRetry; _statisticsTrackerOptions = statisticsTrackerOptions; _sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>(); _readTaskQueue = new AsyncCollection <SocketPayloadReadTask>(); //dedicate a long running task to the read/write operations _socketTask = Task.Run(async() => { await DedicatedSocketTask(); }); _disposeTask = _disposeToken.Token.CreateTask(); _disposeRegistration = _disposeToken.Token.Register(() => { _sendTaskQueue.CompleteAdding(); _readTaskQueue.CompleteAdding(); }); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null) : this(log, endpoint, maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes), null) { }
public IKafkaConnection Create(KafkaEndpoint endpoint, KafkaOptions kafkaOptions) { return(new KafkaConnection(new KafkaTcpSocket(endpoint, kafkaOptions), kafkaOptions.ResponseTimeoutMs, kafkaOptions.Log)); }
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, TimeSpan?maximumReconnectionTimeout = null) { return(new KafkaConnection(new KafkaTcpSocket(log, endpoint, maximumReconnectionTimeout), responseTimeoutMs, log)); }
public BrokerConnectionException(string message, KafkaEndpoint endPoint, Exception innerException) : base(message, endPoint, innerException) { BrokerEndPoint = endPoint; }
public BrokerConnectionException(string message, KafkaEndpoint endPoint, params object[] args) : base(string.Format(message, args), endPoint) { }
public KafkaTcpSocket(KafkaEndpoint endpoint, KafkaOptions kafkaOptions) : this(kafkaOptions.Log, endpoint, kafkaOptions.MaximumReconnectionTimeout, kafkaOptions) { }
public BrokerException(string message, KafkaEndpoint endPoint, params object[] args) : base(string.Format(message, args)) { BrokerEndPoint = endPoint; }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint) { _log = log; _endpoint = endpoint; }
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log) { return(new KafkaConnection(new KafkaTcpSocket(log, endpoint), responseTimeoutMs, log)); }
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, int maxRetry, TimeSpan?maximumReconnectionTimeout = null, StatisticsTrackerOptions statisticsTrackerOptions = null) { KafkaTcpSocket socket = new KafkaTcpSocket(log, endpoint, maxRetry, maximumReconnectionTimeout, statisticsTrackerOptions); return(new KafkaConnection(socket, responseTimeoutMs, log)); }
public NetworkWriteStatistic(KafkaEndpoint endpoint, KafkaDataPayload payload) { CreatedOnUtc = DateTime.UtcNow; Endpoint = endpoint; Payload = payload; }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); }
public KafkaConnectionTests() { _log = new DefaultTraceLog(); _kafkaEndpoint = new DefaultKafkaConnectionFactory().Resolve(new Uri("http://localhost:8999"), _log); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="delayConnectAttemptMS">Time in milliseconds to delay the initial connection attempt to the given server.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, int delayConnectAttemptMS = 0) { _log = log; _endpoint = endpoint; Task.Delay(TimeSpan.FromMilliseconds(delayConnectAttemptMS)).ContinueWith(x => TriggerReconnection()); }