private async Task HandleStartPartition(int partition, PartitionState state, TopicOnOff startInfo) { // Activate partition state.Active = true; state.StopAt = Offsets.Never; // Known offset => FetchRequest if (startInfo.Offset >= 0) { state.NextOffset = startInfo.Offset; await Fetch(startInfo.Topic, partition, startInfo.Offset); } // Restart from last known offset else if (startInfo.Offset == Offsets.Now) { // last known offset was unknown => OffsetRequest if (state.NextOffset < 0) { await Offset(startInfo.Topic, partition, state.NextOffset); } else { await Fetch(startInfo.Topic, partition, state.NextOffset); } } // Unknown offset => OffsetRequest else { await Offset(startInfo.Topic, partition, startInfo.Offset); } }
/// <summary> /// Set the stop mark for a given topic. Depending on the value and the last seen /// offset we either disable the topic/partition now or mark it for later check. /// </summary> /// <param name="topicOnOff"></param> private void HandleStop(TopicOnOff topicOnOff) { if (!_partitionsStates.ContainsKey(topicOnOff.Topic)) { return; } foreach ( var partition in topicOnOff.Partition == Partitions.All ? _partitionsStates[topicOnOff.Topic].Keys : Enumerable.Repeat(topicOnOff.Partition, 1)) { if (!IsAssigned(topicOnOff.Topic, partition)) { // Skip non assigned partitions in case of consumer group usage continue; } PartitionState state; try { state = _partitionsStates[topicOnOff.Topic][partition]; } catch (Exception exception) { // Topic / partition was probably never started InternalError(exception); continue; } if (topicOnOff.Offset == Offsets.Now || (topicOnOff.Offset >= 0 && state.NextOffset > topicOnOff.Offset)) { if (state.Postponed) { state.Postponed = false; state.Active = false; } else { state.StopAt = topicOnOff.Offset == Offsets.Now ? state.NextOffset - 1 : topicOnOff.Offset; if (state.StopAt < 0) { state.StopAt = Offsets.Now; } } } else if (topicOnOff.Offset < 0) { // nothing to do } else { state.StopAt = topicOnOff.Offset; } } }
/// <summary> /// Manage beginning of fetch on given topic/partition. /// If the required offset is positive (i.e. client required a specific known offset /// to begin from) we issue a Fetch request. If the required offset is Earliest or Latest, /// we issue an Offset request to discover the offset values before beginning fetch requests. /// /// This method initializes the PartitionState associated to this topic / partition. /// </summary> /// <param name="topicOnOff"></param> /// <returns></returns> private async Task HandleStart(TopicOnOff topicOnOff) { int[] partitions = null; if (topicOnOff.Partition >= 0) { partitions = _tmpPartitions; partitions[0] = topicOnOff.Partition; } else { while (partitions == null) { try { partitions = await _cluster.RequireAllPartitionsForTopic(topicOnOff.Topic); } catch { // Means the cluster is kind of dead, ignore and retry, there's not much else // to do anyway since we won't be able to send more fetch requests until // some node is available. // TODO: be verbose? } } } foreach (int partition in partitions) { // Initialize PartitionState if needed Dictionary <int, PartitionState> states; if (!_partitionsStates.TryGetValue(topicOnOff.Topic, out states)) { states = new Dictionary <int, PartitionState>(); _partitionsStates[topicOnOff.Topic] = states; } var state = new PartitionState { LastOffsetSeen = Offsets.None, NextOffsetExpected = topicOnOff.Offset, Active = true, Postponed = false }; states[partition] = state; // Known offset => FetchRequest if (topicOnOff.Offset >= 0) { await Fetch(topicOnOff.Topic, partition, topicOnOff.Offset); } // Unknown offset => OffsetRequest else { await Offset(topicOnOff.Topic, partition, topicOnOff.Offset); } } }
/// <summary> /// Set the stop mark for a given topic. Depending on the value and the last seen /// offset we either disable the topic/partition now or mark it for later check. /// </summary> /// <param name="topicOnOff"></param> private void HandleStop(TopicOnOff topicOnOff) { foreach ( var partition in topicOnOff.Partition == Partitions.All ? _partitionsStates[topicOnOff.Topic].Keys : Enumerable.Repeat(topicOnOff.Partition, 1)) { PartitionState state; try { state = _partitionsStates[topicOnOff.Topic][partition]; } catch (Exception exception) { // Topic / partition was probably not started InternalError(exception); continue; } if (topicOnOff.Offset == Offsets.Now || (topicOnOff.Offset >= 0 && state.LastOffsetSeen > topicOnOff.Offset)) { state.Active = false; state.Postponed = false; continue; } if (topicOnOff.Offset < 0) { continue; } state.StopAt = topicOnOff.Offset; } }
/// <summary> /// Manage beginning of fetch on given topic/partition. /// If the required offset is positive (i.e. client required a specific known offset /// to begin from) we issue a Fetch request. If the required offset is Earliest or Latest, /// we issue an Offset request to discover the offset values before beginning fetch requests. /// /// This method initializes the PartitionState associated to this topic / partition. /// </summary> /// <param name="topicOnOff"></param> /// <returns></returns> private async Task HandleStart(TopicOnOff topicOnOff) { int[] partitions = null; if (topicOnOff.Partition >= 0) { partitions = _tmpPartitions; partitions[0] = topicOnOff.Partition; } else { if (_consumerGroup != null) { // (Re)start all assigned partitions partitions = _partitionAssignments[topicOnOff.Topic].Select(po => po.Partition).ToArray(); } else { while (partitions == null) { try { partitions = await _cluster.RequireAllPartitionsForTopic(topicOnOff.Topic); } catch { // Means the cluster is kind of dead, ignore and retry, there's not much else // to do anyway since we won't be able to send more fetch requests until // some node is available. // TODO: be verbose? } } } } foreach (int partition in partitions) { // Initialize PartitionState if needed Dictionary <int, PartitionState> states; if (!_partitionsStates.TryGetValue(topicOnOff.Topic, out states)) { states = new Dictionary <int, PartitionState>(); _partitionsStates[topicOnOff.Topic] = states; } PartitionState state; if (!states.TryGetValue(partition, out state)) { state = new PartitionState { NextOffset = topicOnOff.Offset, Active = false, Postponed = false }; states[partition] = state; } // Already active partitions are not (re)started if (!state.Active) { await HandleStartPartition(partition, state, topicOnOff); } else // But we change back their StopAt mark { state.StopAt = Offsets.Never; } } }