public static void QueueNetworkWrite(KafkaEndpoint endpoint, KafkaDataPayload payload) { if (payload.TrackPayload == false) return; var stat = new NetworkWriteStatistic(endpoint, payload); NetworkWriteQueuedIndex.TryAdd(payload.CorrelationId, stat); Interlocked.Increment(ref Gauges.QueuedWriteOperation); }
public KafkaEndpoint Resolve(Uri kafkaAddress, IKafkaLog log) { var ipAddress = GetFirstAddress(kafkaAddress.Host, log); var ipEndpoint = new IPEndPoint(ipAddress, kafkaAddress.Port); var kafkaEndpoint = new KafkaEndpoint() { ServeUri = kafkaAddress, Endpoint = ipEndpoint }; return kafkaEndpoint; }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); _sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>(); _readTaskQueue = new AsyncCollection<SocketPayloadReadTask>(); //dedicate a long running task to the read/write operations _socketTask = Task.Run(async () => { await DedicatedSocketTask(); }); _disposeTask = _disposeToken.Token.CreateTask(); _disposeRegistration = _disposeToken.Token.Register(() => { _sendTaskQueue.CompleteAdding(); _readTaskQueue.CompleteAdding(); }); }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); _sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>(); _readTaskQueue = new AsyncCollection<SocketPayloadReadTask>(); //dedicate a long running task to the read/write operations _socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); _disposeRegistration = _disposeToken.Token.Register(() => { _sendTaskQueue.CompleteAdding(); _readTaskQueue.CompleteAdding(); }); }
public KafkaConnectionTests() { _log = new DefaultTraceLog(); _kafkaEndpoint = new DefaultKafkaConnectionFactory().Resolve(new Uri("http://localhost:8999"), _log); }
protected bool Equals(KafkaEndpoint other) { return(Equals(Endpoint, other.Endpoint)); }
public KafkaTcpSocketTests() { var log = new DefaultTraceLog(); _fakeServerUrl = new DefaultKafkaConnectionFactory().Resolve(new Uri("http://localhost:8999"), log); _badServerUrl = new DefaultKafkaConnectionFactory().Resolve(new Uri("http://localhost:1"), log); }
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, TimeSpan? maximumReconnectionTimeout = null) { return new KafkaConnection(new KafkaTcpSocket(log, endpoint, maximumReconnectionTimeout), responseTimeoutMs, log); }
private void UpsertConnectionToBrokerConnectionIndex(int brokerId, KafkaEndpoint brokerEndpoint, Func<int, IKafkaConnection> connectionFactory) { //associate the connection with the broker id, and add or update the reference _brokerConnectionIndex.AddOrUpdate(brokerId, connectionFactory, (i, existingConnection) => { //if a connection changes for a broker close old connection and create a new one if (existingConnection.Endpoint.Equals(brokerEndpoint)) return existingConnection; _kafkaOptions.Log.WarnFormat("Broker:{0} Uri changed from:{1} to {2}", brokerId, existingConnection.Endpoint, brokerEndpoint); existingConnection.Dispose(); return connectionFactory(i); }); }
public NetworkWriteStatistic(KafkaEndpoint endpoint, KafkaDataPayload payload) { CreatedOnUtc = DateTime.UtcNow; Endpoint = endpoint; Payload = payload; }
/// <summary> /// Construct socket and open connection to a specified server. /// </summary> /// <param name="log">Logging facility for verbose messaging of actions.</param> /// <param name="endpoint">The IP endpoint to connect to.</param> /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param> public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null) { _log = log; _endpoint = endpoint; _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes); }
public BrokerConnectionException(string message, KafkaEndpoint endPoint, Exception innerException) : base(message, endPoint, innerException) { BrokerEndPoint = endPoint; }
public BrokerConnectionException(string message, KafkaEndpoint endPoint, params object[] args) : base(string.Format(message, args), endPoint) { }
public BrokerException(SerializationInfo info, StreamingContext context) : base(info, context) { bool hasFetch = info.GetByte("HasBrokerEndPoint") == 1; if (hasFetch) { IPEndPoint ipEndPoint = null; Uri uri = null; if (info.GetByte("HasEndpoint") == 1) { ipEndPoint = new IPEndPoint(info.GetInt64("Address"), info.GetInt32("Port")); } if (info.GetByte("HasServeUri") == 1) { uri = new Uri(info.GetString("ServeUri")); } BrokerEndPoint = new KafkaEndpoint { Endpoint = ipEndPoint, ServeUri = uri }; } }
public BrokerException(string message, KafkaEndpoint endPoint, params object[] args) : base(string.Format(message, args)) { BrokerEndPoint = endPoint; }
protected bool Equals(KafkaEndpoint other) { return Equals(Endpoint, other.Endpoint); }