public override long HandleOffsetOutOfRange(TopicAndPartition topicAndPartition)
{
long startTimestamp;
switch (this.config.AutoOffsetReset)
{
case OffsetRequest.SmallestTimeString:
startTimestamp = OffsetRequest.EarliestTime;
break;
case OffsetRequest.LargestTimeString:
startTimestamp = OffsetRequest.LatestTime;
break;
default:
startTimestamp = OffsetRequest.LatestTime;
break;
}
var newOffset = simpleConsumer.EarliestOrLatestOffset(
topicAndPartition, startTimestamp, Request.OrdinaryConsumerId);
var pti = this.partitionMap.Get(topicAndPartition);
pti.ResetFetchOffset(newOffset);
pti.ResetConsumeOffset(newOffset);
return(newOffset);
}
/// <summary>
/// Get the earliest or latest offset of a given topic, partition.
/// </summary>
/// <param name="topicAndPartition">Topic and partition of which the offset is needed.</param>
/// <param name="earliestOrLatest">A value to indicate earliest or latest offset.</param>
/// <param name="consumerId">Id of the consumer which could be a consumer client, SimpleConsumerShell or a follower broker.</param>
/// <returns>Requested offset.</returns>
public long EarliestOrLatestOffset(TopicAndPartition topicAndPartition, long earliestOrLatest, int consumerId)
{
var request =
new OffsetRequest(
new Dictionary <TopicAndPartition, PartitionOffsetRequestInfo>
{
{
topicAndPartition,
new PartitionOffsetRequestInfo(earliestOrLatest, 1)
}
},
clientId: this.ClientId,
replicaId: consumerId);
var partitionErrorAndOffset = this.GetOffsetsBefore(request).PartitionErrorAndOffsets[topicAndPartition];
long offset;
if (partitionErrorAndOffset.Error == ErrorMapping.NoError)
{
offset = partitionErrorAndOffset.Offsets[0];
}
else
{
throw ErrorMapping.ExceptionFor(partitionErrorAndOffset.Error);
}
return(offset);
}
public void ShouldHoldTopicAndPartition()
{
var tp = new TopicAndPartition("testTopic", 1);
tp.PartitionId.Should().Be(1);
tp.Topic.Should().Be("testTopic");
}
public void TopicAndPartitionShouldEqual()
{
var tp = new TopicAndPartition("testTopic", 1);
var tp2 = new TopicAndPartition("testTopic", 1);
tp.Equals(tp2).Should().BeTrue();
tp.Equals(new TopicAndPartition("testTopic2", 1)).Should().BeFalse();
}
public KafkaMessagingClient(IDistributedSearchConfiguration demoCredential, TopicAndPartition topicAndPartition)
{
var bootstrapServers = $"{demoCredential.EventHubName}.servicebus.windows.net:9093";
var saslUsername = "******";
var saslPassword = demoCredential.EventHubConnectionString;
var securityProtocol = SecurityProtocol.SaslSsl;
var saslMechanism = SaslMechanism.Plain;
var groupId = "$Default";
this.producer = new ProducerBuilder <Null, string>(new ProducerConfig
{
BootstrapServers = bootstrapServers,
SecurityProtocol = securityProtocol,
SaslMechanism = saslMechanism,
SaslUsername = saslUsername,
SaslPassword = saslPassword,
// SslCaLocation = cacertlocation,
// Debug = "security,broker,protocol",
})
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(Serializers.Utf8)
.Build();
this.consumer = new ConsumerBuilder <Ignore, string>(new ConsumerConfig
{
BootstrapServers = bootstrapServers,
SecurityProtocol = securityProtocol,
SaslMechanism = saslMechanism,
SaslUsername = saslUsername,
SaslPassword = saslPassword,
GroupId = groupId,
BrokerVersionFallback = "1.0.0",
AutoOffsetReset = AutoOffsetReset.Latest,
// SslCaLocation = cacertlocation,
// Debug = "security,broker,protocol",
})
.SetKeyDeserializer(Deserializers.Ignore)
.SetValueDeserializer(Deserializers.Utf8)
.Build();
this.adminClient = new AdminClientBuilder(new AdminClientConfig
{
BootstrapServers = bootstrapServers,
SecurityProtocol = securityProtocol,
SaslMechanism = saslMechanism,
SaslUsername = saslUsername,
SaslPassword = saslPassword,
}).Build();
this.topicPartition = new Lazy <TopicPartition>(() => new TopicPartition(
topic: topicAndPartition.TopicName,
partition: DeterminePartitionID(this.adminClient, topicAndPartition)));
}
public IDictionary <int, Dictionary <TopicAndPartition, List <KeyedMessage <TKey, Message> > > > PartitionAndCollate(List <KeyedMessage <TKey, Message> > messages)
{
var ret = new Dictionary <int, Dictionary <TopicAndPartition, List <KeyedMessage <TKey, Message> > > >();
try
{
foreach (var message in messages)
{
var topicPartitionsList = this.GetPartitionListForTopic(message);
var partitionIndex = this.GetPartition(message.Topic, message.PartitionKey, topicPartitionsList);
var brokerPartition = topicPartitionsList.ElementAt(partitionIndex);
// postpone the failure until the send operation, so that requests for other brokers are handled correctly
var leaderBrokerId = brokerPartition.LeaderBrokerIdOpt ?? -1;
Dictionary <TopicAndPartition, List <KeyedMessage <TKey, Message> > > dataPerBroker;
if (ret.ContainsKey(leaderBrokerId))
{
dataPerBroker = ret[leaderBrokerId];
}
else
{
dataPerBroker = new Dictionary <TopicAndPartition, List <KeyedMessage <TKey, Message> > >();
ret[leaderBrokerId] = dataPerBroker;
}
var topicAndPartition = new TopicAndPartition(message.Topic, brokerPartition.PartitionId);
List <KeyedMessage <TKey, Message> > dataPerTopicPartition;
if (dataPerBroker.TryGetValue(topicAndPartition, out dataPerTopicPartition) == false)
{
dataPerTopicPartition = new List <KeyedMessage <TKey, Message> >();
dataPerBroker[topicAndPartition] = dataPerTopicPartition;
}
dataPerTopicPartition.Add(message);
}
return(ret);
}
catch (UnknownTopicOrPartitionException e)
{
Logger.Warn("Failed to collate messages by topic,partition due to: " + e.Message, e);
return(null);
}
catch (LeaderNotAvailableException e)
{
Logger.Warn("Failed to collate messages by topic,partition due to: " + e.Message, e);
return(null);
}
catch (Exception e)
{
Logger.Error("Failed to collate messages by topic, partition due to: " + e.Message, e);
return(null);
}
}
public override void ProcessPartitionData(
TopicAndPartition topicAndPartition, long fetchOffset, FetchResponsePartitionData partitionData)
{
var pti = this.partitionMap.Get(topicAndPartition);
if (pti.GetFetchOffset() != fetchOffset)
{
throw new Exception(string.Format("Offset doesn't match for partition [{0},{1}] pti offset: {2} fetch offset: {3}", topicAndPartition.Topic, topicAndPartition.Partiton, pti.GetFetchOffset(), fetchOffset));
}
pti.Enqueue((ByteBufferMessageSet)partitionData.Messages);
}
private FetchResponsePartitionData PartitionDataFor(string topic, int partition)
{
var topicAndPartition = new TopicAndPartition(topic, partition);
FetchResponsePartitionData partitionData;
if (this.Data.TryGetValue(topicAndPartition, out partitionData))
{
return(partitionData);
}
else
{
throw new ArgumentException(string.Format("No partition {0} in fetch response {1}", topicAndPartition, this));
}
}
public SearchServiceStartup(IConfiguration configuration)
{
this.Configuration = configuration;
this.demoCredential = new DemoCredential();
var computeNodeId = 100;
this.responseTopicAndPartition = new TopicAndPartition(
topicName: this.demoCredential.EventHubTopicNameResponses,
partitionSpecification: PartitionSpecification.NewComputeNodeID(computeNodeId));
this.snapshotContainerClient = new BlobContainerClient(
blobContainerUri: new Uri($"https://{this.demoCredential.BusinessDataSnapshotAccountName}.blob.core.windows.net/{this.demoCredential.BusinessDataSnapshotContainerName}/"),
credential: this.demoCredential.AADServicePrincipal);
}
private Dictionary <TopicAndPartition, BufferedMessageSet> GroupMessagesToSet(Dictionary <TopicAndPartition, List <ProducerData <TK, Message> > > eventsPerTopicAndPartition)
{
Dictionary <TopicAndPartition, BufferedMessageSet> messagesPerTopicPartition = new Dictionary <TopicAndPartition, BufferedMessageSet>();
foreach (KeyValuePair <TopicAndPartition, List <ProducerData <TK, Message> > > keyValuePair in eventsPerTopicAndPartition)
{
TopicAndPartition topicAndPartition = keyValuePair.Key;
List <ProducerData <TK, Message> > produceData = keyValuePair.Value;
List <Message> messages = new List <Message>();
produceData.ForEach(p => messages.AddRange(p.Data));
switch (this.producerConfig.CompressionCodec)
{
case CompressionCodecs.NoCompressionCodec:
messagesPerTopicPartition.Add(topicAndPartition,
new BufferedMessageSet(CompressionCodecs.NoCompressionCodec, messages, topicAndPartition.PartitionId));
break;
default:
byte magic = 0;
byte attributes = 0;
foreach (Message m in messages)
{
magic = m.Magic;
attributes = m.Attributes;
m.CleanMagicAndAttributesBeforeCompress();
}
if (!this.producerConfig.CompressedTopics.Any() || this.producerConfig.CompressedTopics.Contains(topicAndPartition.Topic))
{
messagesPerTopicPartition.Add(topicAndPartition, new BufferedMessageSet(this.producerConfig.CompressionCodec, messages, topicAndPartition.PartitionId));
}
else
{
messagesPerTopicPartition.Add(topicAndPartition, new BufferedMessageSet(CompressionCodecs.NoCompressionCodec, messages, topicAndPartition.PartitionId));
}
foreach (Message m in messages)
{
m.RestoreMagicAndAttributesAfterCompress(magic, attributes);
}
break;
}
}
return(messagesPerTopicPartition);
}
/// <summary>
/// process fetched Data
/// </summary>
/// <param name="topicAndPartition"></param>
/// <param name="fetchOffset"></param>
/// <param name="partitionData"></param>
public abstract void ProcessPartitionData(
TopicAndPartition topicAndPartition, long fetchOffset, FetchResponsePartitionData partitionData);
public static IRequestResponseMessageClient <T> Responses <T>(IDistributedSearchConfiguration demoCredential, TopicAndPartition topicAndPartition) => new KafkaMessagingClient <T>(demoCredential: demoCredential, topicAndPartition: topicAndPartition);
/// <summary>
/// Given the message to be pushed, return the partition selected, the broker leader for the partition
/// </summary>
/// <param name="events">message set to be produced</param>
/// <returns>the partition selected and the broker leader</returns>
private IEnumerable <KeyValuePair <int, Dictionary <TopicAndPartition, List <ProducerData <TK, Message> > > > > PartitionAndCollate(IEnumerable <ProducerData <TK, Message> > events)
{
var ret = new Dictionary <int, Dictionary <TopicAndPartition, List <ProducerData <TK, Message> > > >();
if (this.producerConfig.ForceToPartition >= 0)
{
int leaderBrokerId = this.producerConfig.Brokers[0].BrokerId;
Dictionary <TopicAndPartition, List <ProducerData <TK, Message> > > dataPerBroker = new Dictionary <TopicAndPartition, List <ProducerData <TK, Message> > >();
dataPerBroker.Add(new TopicAndPartition(events.First().Topic, this.producerConfig.ForceToPartition), events.ToList());
if (producerConfig.Verbose)
{
Logger.DebugFormat("PartitionAndCollate ForceToPartition ,totalNumPartitions={0},ForceToPartition={1},leaderBrokerId={2}",
0, this.producerConfig.ForceToPartition, leaderBrokerId);
}
ret.Add(leaderBrokerId, dataPerBroker);
}
else
{
foreach (var eventItem in events)
{
//Sorted list of all partition, some has leader, some not.
List <Partition> topicPartitionsList = this.GetPartitionListForTopic(eventItem);
// when the total number of partitions is specified in the ProducerConf, do not check for the number of active partitions again,
// this ensures the partition selected per the pre-determined number of partitions, instead of actually number of partitions checked at run-time
var totalNumPartitions = this.producerConfig.TotalNumPartitions == 0 ? topicPartitionsList.Count() : this.producerConfig.TotalNumPartitions;
var partitionIndex = this.GetPartition(eventItem.Key, eventItem.IsKeyNull, totalNumPartitions);
// when the total number of partition is specified, this.GetPartitionListForTopic() returns only one partition corresponding to the partitionIndex
Partition brokerPartition = this.producerConfig.TotalNumPartitions == 0 ? topicPartitionsList.ElementAt(partitionIndex) : topicPartitionsList[0];
var leaderBrokerId = brokerPartition.Leader != null ? brokerPartition.Leader.BrokerId : -1; // postpone the failure until the send operation, so that requests for other brokers are handled correctly
if (producerConfig.Verbose)
{
Logger.DebugFormat("PartitionAndCollate,totalNumPartitions={0},eventItem.Key={1},partitionIndex={2},brokerPartition={3},leaderBrokerId={4}",
totalNumPartitions, eventItem.Key, partitionIndex, brokerPartition, leaderBrokerId);
}
if (leaderBrokerId == -1)
{
Logger.WarnFormat("No leader for partition {0} brokerPartition:{1} ", partitionIndex, brokerPartition.ToString());
}
Dictionary <TopicAndPartition, List <ProducerData <TK, Message> > > dataPerBroker = null;
if (ret.ContainsKey(leaderBrokerId))
{
dataPerBroker = ret[leaderBrokerId];
}
else
{
dataPerBroker = new Dictionary <TopicAndPartition, List <ProducerData <TK, Message> > >();
ret.Add(leaderBrokerId, dataPerBroker);
}
var topicAndPartition = new TopicAndPartition(eventItem.Topic, brokerPartition.PartId);
List <ProducerData <TK, Message> > dataPerTopicPartition = null;
if (dataPerBroker.ContainsKey(topicAndPartition))
{
dataPerTopicPartition = dataPerBroker[topicAndPartition];
}
else
{
dataPerTopicPartition = new List <ProducerData <TK, Message> >();
dataPerBroker.Add(topicAndPartition, dataPerTopicPartition);
}
dataPerTopicPartition.Add(eventItem);
}
}
return(ret);
}
private IObservable <RequestResponseMessage <ProviderSearchResponse <T> > > CreateProviderResponsePump <T>(TopicAndPartition topicAndPartition)
{
var messagingClient = MessagingClients
.Responses <ProviderSearchResponse <T> >(
demoCredential: this.demoCredential,
topicAndPartition: topicAndPartition);
var connectable = messagingClient
.CreateObervable()
.Publish();
connectable.Connect();
return(connectable.AsObservable());
}
/// <summary> /// handle a partition whose offset is out of range and return a new fetch offset /// </summary> /// <param name="topicAndPartition"></param> /// <returns></returns> public abstract long HandleOffsetOutOfRange(TopicAndPartition topicAndPartition);
public override void DoWork()
{
var leaderForPartitionsMap = new Dictionary <TopicAndPartition, Broker>();
[email protected]();
try
{
while (this.parent.NoLeaderPartitionSet.Count == 0)
{
Logger.Debug("No partition for leader election.");
this.parent.cond.Await();
}
Logger.DebugFormat("Partitions without leader {0}", string.Join(",", this.parent.NoLeaderPartitionSet));
var brokers = ZkUtils.GetAllBrokersInCluster(this.parent.zkClient);
var topicsMetadata =
ClientUtils.FetchTopicMetadata(
new HashSet <string>(this.parent.NoLeaderPartitionSet.Select(m => m.Topic)),
brokers,
this.parent.config.ClientId,
this.parent.config.SocketTimeoutMs,
this.parent.correlationId.GetAndIncrement()).TopicsMetadata;
if (Logger.IsDebugEnabled)
{
foreach (var topicMetadata in topicsMetadata)
{
Logger.Debug(topicMetadata);
}
}
foreach (var tmd in topicsMetadata)
{
var topic = tmd.Topic;
foreach (var pmd in tmd.PartitionsMetadata)
{
var topicAndPartition = new TopicAndPartition(topic, pmd.PartitionId);
if (pmd.Leader != null && this.parent.NoLeaderPartitionSet.Contains(topicAndPartition))
{
var leaderBroker = pmd.Leader;
leaderForPartitionsMap[topicAndPartition] = leaderBroker;
this.parent.NoLeaderPartitionSet.Remove(topicAndPartition);
}
}
}
}
catch (Exception e)
{
if (!isRunning.Get())
{
throw; /* If this thread is stopped, propagate this exception to kill the thread. */
}
else
{
Logger.Warn("Failed to find leader for " + string.Join(",", this.parent.NoLeaderPartitionSet), e);
}
}
finally
{
[email protected]();
}
try
{
this.parent.AddFetcherForPartitions(
leaderForPartitionsMap.ToDictionary(
kvp => kvp.Key,
kvp =>
new BrokerAndInitialOffset(kvp.Value, this.parent.partitionMap.Get(kvp.Key).GetFetchOffset())));
}
catch (Exception e)
{
if (!isRunning.Get())
{
throw; /* If this thread is stopped, propagate this exception to kill the thread. */
}
else
{
Logger.Warn(string.Format("Failed to add leader for partitions {0}; will retry", string.Join(",", leaderForPartitionsMap.Keys)), e);
[email protected]();
foreach (var leader in leaderForPartitionsMap.Keys)
{
this.parent.NoLeaderPartitionSet.Add(leader);
}
[email protected]();
}
}
this.parent.ShutdownIdleFetcherThreads();
Thread.Sleep(this.parent.config.RefreshLeaderBackoffMs);
}
private static ConfluentPartition DeterminePartitionID(IAdminClient adminClient, TopicAndPartition topicAndPartition)
{
MercuryPartition partitionId = determinePartitionID(
determinePartitionCount: GetPartitionCount(adminClient).ToFSharpFunc(),
topicAndPartition: topicAndPartition);
return(partitionId switch
{
MercuryPartition.Partition x => new ConfluentPartition(x.Item),
_ => ConfluentPartition.Any,
});