public async Task ConnectAsync()
{
// if we've already connected, don't do it again.
if (IsConnected)
{
return;
}
await _cluster.Scheduler.Ask(async() =>
{
await _sync.WaitAsync();
try
{
if (IsConnected)
{
return;
}
_log.Debug("Connecting producer {1} for topic {0}", Topic, _id);
EtwTrace.Log.ProducerStarting(Topic, _id);
_sendMessagesSubject = new Subject <Message>();
if (_cluster.State != Cluster.ClusterState.Connected)
{
_log.Debug("Connecting cluster");
await _cluster.ConnectAsync();
_log.Debug("Connected cluster");
}
// Recovery: subscribe to partition offline/online events
_cluster.PartitionStateChanges.
Where(p => p.Topic == Configuration.Topic).
Synchronize(_allPartitionQueues).
Subscribe(p =>
{
PartitionQueueInfo queue;
if (!_allPartitionQueues.TryGetValue(p.PartitionId, out queue))
{
queue = new PartitionQueueInfo(Configuration.SendBuffersInitialSize)
{
Partition = p.PartitionId
};
_allPartitionQueues.Add(p.PartitionId, queue);
}
_log.Info("Detected change in topic/partition '{0}'/{1}/{2} IsOnline {3}->{4}",
Configuration.Topic,
p.PartitionId,
p.ErrorCode,
queue.IsOnline,
p.ErrorCode.IsSuccess());
queue.IsOnline = p.ErrorCode.IsSuccess();
_queueEventWaitHandler.Set();
});
// get all of the partitions for this topic. Allows the MessagePartitioner to select a partition.
var topicPartitions = await _cluster.GetOrFetchMetaForTopicAsync(Configuration.Topic);
_log.Debug("Producer found {0} partitions for '{1}'", topicPartitions.Length, Configuration.Topic);
_sendMessagesSubject.
Do(msg => msg.PartitionId = Configuration.Partitioner.GetMessagePartition(msg, topicPartitions).Id).
Buffer(Configuration.BatchFlushTime, Configuration.BatchFlushSize).
Where(b => b.Count > 0).
Select(msgs => msgs.GroupBy(msg => msg.PartitionId)).
ObserveOn(_cluster.Scheduler).
Subscribe(partitionGroups =>
{
foreach (var batch in partitionGroups)
{
// partition queue might be created if metadata broadcast fired already
PartitionQueueInfo queue;
if (!_allPartitionQueues.TryGetValue(batch.Key, out queue))
{
queue = new PartitionQueueInfo(Configuration.SendBuffersInitialSize)
{
Partition = batch.Key
};
_allPartitionQueues.Add(batch.Key, queue);
queue.IsOnline = topicPartitions.First(p => p.Id == batch.Key).ErrorCode.IsSuccess();
_log.Debug("{0} added new partition queue", this);
}
// now queue them up
var batchAr = batch.ToArray();
// make sure we have space.
if (queue.Queue.Size + batchAr.Length > queue.Queue.Capacity)
{
// try to increase the capacity.
if (Configuration.AutoGrowSendBuffers)
{
var growBy = Math.Max(queue.Queue.Capacity / 2 + 1, 2 * batchAr.Length);
_log.Warn("Capacity of send buffer with size {3} not large enough to accept {2} new messages. Increasing capacity from {0} to {1}", queue.Queue.Capacity, queue.Queue.Capacity + growBy, batchAr.Length, queue.Queue.Size);
queue.Queue.Capacity += growBy;
}
else
{
// we're full and not allowed to grow. Throw the batch back to the caller, and continue on.
var msg = string.Format("Send Buffer Full for partition {0}. ",
Cluster.PartitionStateChanges.Where(
ps => ps.Topic == Configuration.Topic && ps.PartitionId == queue.Partition)
.Take(1)
.Wait());
_log.Error(msg);
if (OnPermError != null)
{
OnPermError(
new Exception(msg)
, batchAr);
}
continue;
}
}
// we have the space, add to the queue
queue.Queue.Put(batchAr);
if (_log.IsDebugEnabled)
{
_log.Debug("Enqueued batch of size {0} for topic '{1}' partition {2}",
batchAr.Length, Configuration.Topic, batch.Key);
}
// After batch enqueued, send wake up signal to sending queue
_queueEventWaitHandler.Set();
}
}, e => _log.Fatal(e, "Error in _sendMessagesSubject pipeline"),
() => _log.Debug("_sendMessagesSubject complete")
);
// start the send loop task
_cluster.Scheduler.Schedule(() => {
_sendLoopTask = SendLoop().
ContinueWith(t =>
{
if (t.IsFaulted)
{
_log.Fatal(t.Exception, "SendLoop failed");
}
else
{
_log.Debug("SendLoop complete with status: {0}", t.Status);
}
});
});
_log.Debug("Connected");
EtwTrace.Log.ProducerStarted(Topic, _id);
}
catch (Exception e)
{
_log.Error(e, "Exception during connect");
EtwTrace.Log.ProducerError(e.Message, _id);
throw;
}
finally
{
_log.Debug("#{0} Releasing Producer Semaphore.", _id);
_sync.Release();
}
}).ConfigureAwait(false);
}
public async Task ConnectAsync()
{
// if we've already connected, don't do it again.
if (IsConnected)
return;
await await _cluster.Scheduler.Ask(async () =>
{
await _sync.WaitAsync();
try
{
if (IsConnected)
return;
_log.Debug("Connecting producer {1} for topic {0}", Topic, _id);
EtwTrace.Log.ProducerStarting(Topic, _id);
_sendMessagesSubject = new Subject<Message>();
if (_cluster.State != Cluster.ClusterState.Connected)
{
_log.Debug("Connecting cluster");
await _cluster.ConnectAsync();
_log.Debug("Connected cluster");
}
// Recovery: subscribe to partition offline/online events
_partitionStateSubsctiption = _cluster.PartitionStateChanges.
Where(p => p.Topic == Configuration.Topic).
Synchronize(_allPartitionQueues).
Subscribe(p =>
{
PartitionQueueInfo queue;
if (!_allPartitionQueues.TryGetValue(p.PartitionId, out queue))
{
queue = new PartitionQueueInfo(Configuration.SendBuffersInitialSize) { Partition = p.PartitionId };
_allPartitionQueues.Add(p.PartitionId, queue);
_queueSizeEvents.OnNext(new QueueResizeInfo{PartitionId = p.PartitionId, Size = queue.Queue.Size, Capacity = queue.Queue.Capacity});
}
_log.Info("Detected change in topic/partition '{0}'/{1}/{2} IsOnline {3}->{4}",
Configuration.Topic,
p.PartitionId,
p.ErrorCode,
queue.IsOnline,
p.ErrorCode.IsSuccess());
queue.IsOnline = p.ErrorCode.IsSuccess();
_queueEventWaitHandler.Set();
});
// get all of the partitions for this topic. Allows the MessagePartitioner to select a partition.
var topicPartitions = await _cluster.GetOrFetchMetaForTopicAsync(Configuration.Topic);
_log.Debug("Producer found {0} partitions for '{1}'", topicPartitions.Length, Configuration.Topic);
_sendMessagesSubject.
Do(msg => msg.PartitionId = Configuration.Partitioner.GetMessagePartition(msg, topicPartitions).Id).
Buffer(Configuration.BatchFlushTime, Configuration.BatchFlushSize).
Where(b => b.Count > 0).
Select(msgs => msgs.GroupBy(msg => msg.PartitionId)).
ObserveOn(_cluster.Scheduler).
Subscribe(partitionGroups =>
{
foreach (var batch in partitionGroups)
{
// partition queue might be created if metadata broadcast fired already
PartitionQueueInfo queue;
if (!_allPartitionQueues.TryGetValue(batch.Key, out queue))
{
queue = new PartitionQueueInfo(Configuration.SendBuffersInitialSize) { Partition = batch.Key };
_allPartitionQueues.Add(batch.Key, queue);
queue.IsOnline = topicPartitions.First(p => p.Id == batch.Key).ErrorCode.IsSuccess();
_log.Debug("{0} added new partition queue", this);
_queueSizeEvents.OnNext(new QueueResizeInfo { PartitionId = queue.Partition, Size = queue.Queue.Size, Capacity = queue.Queue.Capacity });
}
// now queue them up
var batchAr = batch.ToArray();
// make sure we have space.
if (queue.Queue.Size + batchAr.Length > queue.Queue.Capacity)
{
// try to increase the capacity.
if (Configuration.AutoGrowSendBuffers)
{
var growBy = Math.Max(queue.Queue.Capacity/2 + 1, 2 * batchAr.Length);
_log.Warn("Capacity of send buffer with size {3} not large enough to accept {2} new messages. Increasing capacity from {0} to {1}", queue.Queue.Capacity, queue.Queue.Capacity+growBy, batchAr.Length, queue.Queue.Size);
queue.Queue.Capacity += growBy;
_queueSizeEvents.OnNext(new QueueResizeInfo { PartitionId = queue.Partition, Size = queue.Queue.Size, Capacity = queue.Queue.Capacity });
}
else
{
// we're full and not allowed to grow. Throw the batch back to the caller, and continue on.
var msg = string.Format("Send Buffer Full for partition {0}. ",
Cluster.PartitionStateChanges.Where(
ps => ps.Topic == Configuration.Topic && ps.PartitionId == queue.Partition)
.Take(1)
.Wait());
_log.Error(msg);
if (OnPermError != null)
OnPermError(new Exception(msg), batchAr);
continue;
}
}
// we have the space, add to the queue
queue.Queue.Put(batchAr);
if (_log.IsDebugEnabled)
_log.Debug("Enqueued batch of size {0} for topic '{1}' partition {2}",
batchAr.Length, Configuration.Topic, batch.Key);
// After batch enqueued, send wake up signal to sending queue
_queueEventWaitHandler.Set();
}
}, e => _log.Fatal(e, "Error in _sendMessagesSubject pipeline"),
() => _log.Debug("_sendMessagesSubject complete")
);
// start the send loop task
_cluster.Scheduler.Schedule(() => {
_sendLoopTask = SendLoop().
ContinueWith(t =>
{
if (t.IsFaulted)
_log.Fatal(t.Exception, "SendLoop failed");
else
_log.Debug("SendLoop complete with status: {0}", t.Status);
});
});
_log.Debug("Connected");
EtwTrace.Log.ProducerStarted(Topic, _id);
}
catch (Exception e)
{
_log.Error(e, "Exception during connect");
EtwTrace.Log.ProducerError(e.Message, _id);
throw;
}
finally
{
_log.Debug("#{0} Releasing Producer Semaphore.", _id);
_sync.Release();
}
}).ConfigureAwait(false);
}