private async Task HandleStartPartition(int partition, PartitionState state, TopicOnOff startInfo)
{
// Activate partition
state.Active = true;
state.StopAt = Offsets.Never;
// Known offset => FetchRequest
if (startInfo.Offset >= 0)
{
state.NextOffset = startInfo.Offset;
await Fetch(startInfo.Topic, partition, startInfo.Offset);
}
// Restart from last known offset
else if (startInfo.Offset == Offsets.Now)
{
// last known offset was unknown => OffsetRequest
if (state.NextOffset < 0)
{
await Offset(startInfo.Topic, partition, state.NextOffset);
}
else
{
await Fetch(startInfo.Topic, partition, state.NextOffset);
}
}
// Unknown offset => OffsetRequest
else
{
await Offset(startInfo.Topic, partition, startInfo.Offset);
}
}
/// <summary>
/// Set the stop mark for a given topic. Depending on the value and the last seen
/// offset we either disable the topic/partition now or mark it for later check.
/// </summary>
/// <param name="topicOnOff"></param>
private void HandleStop(TopicOnOff topicOnOff)
{
if (!_partitionsStates.ContainsKey(topicOnOff.Topic))
{
return;
}
foreach (
var partition in
topicOnOff.Partition == Partitions.All
? _partitionsStates[topicOnOff.Topic].Keys
: Enumerable.Repeat(topicOnOff.Partition, 1))
{
if (!IsAssigned(topicOnOff.Topic, partition))
{
// Skip non assigned partitions in case of consumer group usage
continue;
}
PartitionState state;
try
{
state = _partitionsStates[topicOnOff.Topic][partition];
}
catch (Exception exception)
{
// Topic / partition was probably never started
InternalError(exception);
continue;
}
if (topicOnOff.Offset == Offsets.Now || (topicOnOff.Offset >= 0 && state.NextOffset > topicOnOff.Offset))
{
if (state.Postponed)
{
state.Postponed = false;
state.Active = false;
}
else
{
state.StopAt = topicOnOff.Offset == Offsets.Now ? state.NextOffset - 1 : topicOnOff.Offset;
if (state.StopAt < 0)
{
state.StopAt = Offsets.Now;
}
}
}
else if (topicOnOff.Offset < 0)
{
// nothing to do
}
else
{
state.StopAt = topicOnOff.Offset;
}
}
}
/// <summary>
/// Manage beginning of fetch on given topic/partition.
/// If the required offset is positive (i.e. client required a specific known offset
/// to begin from) we issue a Fetch request. If the required offset is Earliest or Latest,
/// we issue an Offset request to discover the offset values before beginning fetch requests.
///
/// This method initializes the PartitionState associated to this topic / partition.
/// </summary>
/// <param name="topicOnOff"></param>
/// <returns></returns>
private async Task HandleStart(TopicOnOff topicOnOff)
{
int[] partitions = null;
if (topicOnOff.Partition >= 0)
{
partitions = _tmpPartitions;
partitions[0] = topicOnOff.Partition;
}
else
{
while (partitions == null)
{
try
{
partitions = await _cluster.RequireAllPartitionsForTopic(topicOnOff.Topic);
}
catch
{
// Means the cluster is kind of dead, ignore and retry, there's not much else
// to do anyway since we won't be able to send more fetch requests until
// some node is available.
// TODO: be verbose?
}
}
}
foreach (int partition in partitions)
{
// Initialize PartitionState if needed
Dictionary <int, PartitionState> states;
if (!_partitionsStates.TryGetValue(topicOnOff.Topic, out states))
{
states = new Dictionary <int, PartitionState>();
_partitionsStates[topicOnOff.Topic] = states;
}
var state = new PartitionState
{
LastOffsetSeen = Offsets.None,
NextOffsetExpected = topicOnOff.Offset,
Active = true,
Postponed = false
};
states[partition] = state;
// Known offset => FetchRequest
if (topicOnOff.Offset >= 0)
{
await Fetch(topicOnOff.Topic, partition, topicOnOff.Offset);
}
// Unknown offset => OffsetRequest
else
{
await Offset(topicOnOff.Topic, partition, topicOnOff.Offset);
}
}
}
/// <summary>
/// Set the stop mark for a given topic. Depending on the value and the last seen
/// offset we either disable the topic/partition now or mark it for later check.
/// </summary>
/// <param name="topicOnOff"></param>
private void HandleStop(TopicOnOff topicOnOff)
{
foreach (
var partition in
topicOnOff.Partition == Partitions.All
? _partitionsStates[topicOnOff.Topic].Keys
: Enumerable.Repeat(topicOnOff.Partition, 1))
{
PartitionState state;
try
{
state = _partitionsStates[topicOnOff.Topic][partition];
}
catch (Exception exception)
{
// Topic / partition was probably not started
InternalError(exception);
continue;
}
if (topicOnOff.Offset == Offsets.Now ||
(topicOnOff.Offset >= 0 && state.LastOffsetSeen > topicOnOff.Offset))
{
state.Active = false;
state.Postponed = false;
continue;
}
if (topicOnOff.Offset < 0)
{
continue;
}
state.StopAt = topicOnOff.Offset;
}
}
/// <summary>
/// Manage beginning of fetch on given topic/partition.
/// If the required offset is positive (i.e. client required a specific known offset
/// to begin from) we issue a Fetch request. If the required offset is Earliest or Latest,
/// we issue an Offset request to discover the offset values before beginning fetch requests.
///
/// This method initializes the PartitionState associated to this topic / partition.
/// </summary>
/// <param name="topicOnOff"></param>
/// <returns></returns>
private async Task HandleStart(TopicOnOff topicOnOff)
{
int[] partitions = null;
if (topicOnOff.Partition >= 0)
{
partitions = _tmpPartitions;
partitions[0] = topicOnOff.Partition;
}
else
{
if (_consumerGroup != null)
{
// (Re)start all assigned partitions
partitions = _partitionAssignments[topicOnOff.Topic].Select(po => po.Partition).ToArray();
}
else
{
while (partitions == null)
{
try
{
partitions = await _cluster.RequireAllPartitionsForTopic(topicOnOff.Topic);
}
catch
{
// Means the cluster is kind of dead, ignore and retry, there's not much else
// to do anyway since we won't be able to send more fetch requests until
// some node is available.
// TODO: be verbose?
}
}
}
}
foreach (int partition in partitions)
{
// Initialize PartitionState if needed
Dictionary <int, PartitionState> states;
if (!_partitionsStates.TryGetValue(topicOnOff.Topic, out states))
{
states = new Dictionary <int, PartitionState>();
_partitionsStates[topicOnOff.Topic] = states;
}
PartitionState state;
if (!states.TryGetValue(partition, out state))
{
state = new PartitionState {
NextOffset = topicOnOff.Offset, Active = false, Postponed = false
};
states[partition] = state;
}
// Already active partitions are not (re)started
if (!state.Active)
{
await HandleStartPartition(partition, state, topicOnOff);
}
else // But we change back their StopAt mark
{
state.StopAt = Offsets.Never;
}
}
}
