public void Awake(){
// Populate the partition interactable parent dictionary, only for ones existing
foreach(Transform trans in partitionParent){
PartitionMetadata metadata = trans.GetComponent<PartitionMetadata>();
if(metadata != null){
partitionInteractableDictionary.Add(metadata.partitionNumber, metadata.interactables);
}
else{
Debug.LogError("Non partition detected " + trans.name);
}
}
}
public void Constuctor()
{
int[] replicas = new int[0];
int[] inSyncReplicas = new int[0];
var pm = new PartitionMetadata(33, 1, replicas, inSyncReplicas, ErrorCode.Local_AllBrokersDown);
Assert.Equal(pm.PartitionId, 33);
Assert.Equal(pm.Leader, 1);
Assert.Same(pm.Replicas, replicas);
Assert.Same(pm.InSyncReplicas, inSyncReplicas);
Assert.Equal(pm.Error, new Error(ErrorCode.Local_AllBrokersDown));
}
public void Constuctor()
{
int[] replicas = new int[0];
int[] inSyncReplicas = new int[0];
var pm = new PartitionMetadata(33, 1, replicas, inSyncReplicas, ErrorCode._ALL_BROKERS_DOWN);
Assert.Equal(pm.PartitionId, 33);
Assert.Equal(pm.Leader, 1);
Assert.Same(pm.Replicas, replicas);
Assert.Same(pm.InSyncReplicas, inSyncReplicas);
Assert.Equal(pm.Error, new Error(ErrorCode._ALL_BROKERS_DOWN));
}
public int Parse(byte[] data, int dataOffset)
{
int bufferOffset = dataOffset;
bufferOffset = BufferReader.Read(data, bufferOffset, out topicErrorCode);
bufferOffset = BufferReader.Read(data, bufferOffset, out topicName);
PartitionMetadatas = new Dictionary<int, PartitionMetadata>();
int partitionCount;
bufferOffset = BufferReader.Read(data, bufferOffset, out partitionCount);
for (var i = 0; i < partitionCount; i++)
{
PartitionMetadata metadata = new PartitionMetadata();
bufferOffset = metadata.Parse(data, bufferOffset);
PartitionMetadatas.Add(metadata.PartitionId, metadata);
}
return bufferOffset;
}
public async Task CanLoadWatermarksWithValidParamsAsync()
{
// Arrange
var topic = new TopicName("test");
var clientMock = new Mock <IAdminClient>();
var client = clientMock.Object;
var timeout = 1000;
var loader = new TopicWatermarkLoader(topic, client, timeout);
var consumerMock = new Mock <IConsumer <object, object> >();
IConsumer <object, object> consumerFactory() => consumerMock.Object;
var adminClientPartition = new TopicPartition(topic.Value, new Partition(1));
var adminParitions = new[] { adminClientPartition };
var borkerMeta = new BrokerMetadata(1, "testHost", 1000);
var partitionMeta = new PartitionMetadata(1, 1, new[] { 1 }, new[] { 1 }, null);
var topicMeta = new TopicMetadata(topic.Value, new[] { partitionMeta }.ToList(), null);
var meta = new Confluent.Kafka.Metadata(
new[] { borkerMeta }.ToList(),
new[] { topicMeta }.ToList(), 1, "test"
);
clientMock.Setup(c => c.GetMetadata(topic.Value, TimeSpan.FromSeconds(timeout))).Returns(meta);
var offets = new WatermarkOffsets(new Offset(1), new Offset(2));
consumerMock.Setup(x => x.QueryWatermarkOffsets(adminClientPartition, TimeSpan.FromSeconds(timeout))).Returns(offets);
TopicWatermark result = null !;
// Act
var exception = await Record.ExceptionAsync(async() => result = await loader.LoadWatermarksAsync(consumerFactory, CancellationToken.None));
// Assert
exception.Should().BeNull();
consumerMock.Verify(x => x.Close(), Times.Once);
consumerMock.Verify(x => x.Dispose(), Times.Once);
result.Should().NotBeNull();
var watermarks = result.Watermarks.ToList();
watermarks.Should().ContainSingle();
clientMock.Verify(c => c.GetMetadata(topic.Value, TimeSpan.FromSeconds(timeout)), Times.Once);
consumerMock.Verify(x => x.QueryWatermarkOffsets(adminClientPartition, TimeSpan.FromSeconds(timeout)), Times.Once);
watermarks.Single().TopicName.Should().Be(topic);
watermarks.Single().Partition.Value.Should().Be(partitionMeta.PartitionId);
watermarks.Single().Offset.Should().Be(offets);
}
public int InsertPartitionMetadata(int exhibitId, int mediaId)
{
return(PartitionMetadata.InsertPartitionMetadata(exhibitId, 1, mediaId, "", "IntegrationTest", FileSystemType.FAT));
}
private async Task Fetch(ChannelWriter <Message <TKey, TValue> > messageWriter, string topic, PartitionMetadata partition, BrokerConnection broker, CancellationToken cancellationToken)
{
long offset = 0;
while (!cancellationToken.IsCancellationRequested)
{
var fetch = await broker.Fetch(new FetchRequest
{
ReplicaId = -1,
MaxWaitTime = 100,
MinBytes = 1,
MaxBytes = 64 * 1024,
IsolationLevel = 0,
SessionId = 0,
SessionEpoc = -1,
Topics = new []
{
new FetchRequest.Topic
{
TopicName = topic,
Partitions = new []
{
new FetchRequest.Topic.Partition
{
PartitionId = partition.PartitionId,
CurrentLeaderEpoc = -1,
FetchOffset = offset,
LogStartOffset = -1,
PartitionMaxBytes = 32 * 1024
}
}
}
}
});
long maxoffset = 0;
long high = 0;
foreach (var r in fetch.Responses)
{
foreach (var pr in r.PartitionResponses)
{
high = pr.HighWaterMark;
foreach (var batch in pr.RecordBatches)
{
foreach (var rec in batch.Records)
{
if (rec.Offset > maxoffset)
{
maxoffset = rec.Offset;
}
var msg = new Message <TKey, TValue>
{
Timestamp = Timestamp.UnixTimestampMsToDateTime(batch.FirstTimeStamp + rec.TimeStampDelta),
Topic = r.TopicName,
PartitionId = pr.PartitionId,
Offset = rec.Offset,
Key = await keyDeserializer.Deserialize(rec.Key),
Value = await valueDeserializer.Deserialize(rec.Value)
};
if (rec.Headers.Count > 0)
{
foreach (var h in rec.Headers)
{
msg.AddHeader(h.Key, h.Value);
}
}
await messageWriter.WriteAsync(msg, cancellationToken);
}
}
}
}
if (maxoffset > 0)
{
offset = maxoffset + 1;
}
if (offset >= high)
{
await Task.Delay(200); // linger
}
}
}
private async Task Fetch(ChannelWriter <RecordBatch.Record> messageWriter, string topic, PartitionMetadata partition, BrokerConnection broker, CancellationToken cancellationToken)
{
long offset = 0;
while (!cancellationToken.IsCancellationRequested)
{
var fetch = await broker.Fetch(new FetchRequest
{
ReplicaId = -1,
MaxWaitTime = 100,
MinBytes = 1,
MaxBytes = 64 * 1024,
IsolationLevel = 0,
SessionId = 0,
SessionEpoc = -1,
Topics = new []
{
new FetchRequest.Topic
{
TopicName = topic,
Partitions = new []
{
new FetchRequest.Topic.Partition
{
PartitionId = partition.PartitionId,
CurrentLeaderEpoc = -1,
FetchOffset = offset,
LogStartOffset = -1,
PartitionMaxBytes = 32 * 1024
}
}
}
}
});
long maxoffset = 0;
long high = 0;
foreach (var r in fetch.Responses)
{
foreach (var pr in r.PartitionResponses)
{
high = pr.HighWaterMark;
foreach (var batch in pr.RecordBatches)
{
foreach (var rec in batch.Records)
{
if (rec.Offset > maxoffset)
{
maxoffset = rec.Offset;
}
await messageWriter.WriteAsync(rec, cancellationToken);
}
}
}
}
if (maxoffset > 0)
{
offset = maxoffset + 1;
}
if (offset >= high)
{
await Task.Delay(200); // linger
}
}
}
public void TestPartitionAndCollateEvents()
{
var producerDataList = new List <KeyedMessage <int, Message> >();
// use bogus key and partition key override for some messages
producerDataList.Add(new KeyedMessage <int, Message>("topic1", 0, new Message(Encoding.UTF8.GetBytes("msg1"))));
producerDataList.Add(new KeyedMessage <int, Message>("topic2", -99, 1, new Message(Encoding.UTF8.GetBytes("msg2"))));
producerDataList.Add(new KeyedMessage <int, Message>("topic1", 2, new Message(Encoding.UTF8.GetBytes("msg3"))));
producerDataList.Add(new KeyedMessage <int, Message>("topic1", -101, 3, new Message(Encoding.UTF8.GetBytes("msg4"))));
producerDataList.Add(new KeyedMessage <int, Message>("topic2", 4, new Message(Encoding.UTF8.GetBytes("msg5"))));
var broker1 = new Broker(0, "localhost", 9092);
var broker2 = new Broker(1, "localhost", 9093);
// form expected partitions metadata
var partition1Metadata = new PartitionMetadata(0, broker1, new List <Broker> {
broker1, broker2
});
var partition2Metadata = new PartitionMetadata(1, broker2, new List <Broker> {
broker1, broker2
});
var topic1Metadata = new TopicMetadata(
"topic1", new List <PartitionMetadata> {
partition1Metadata, partition2Metadata
});
var topic2Metadata = new TopicMetadata(
"topic2", new List <PartitionMetadata> {
partition1Metadata, partition2Metadata
});
var topicPartitionInfos = new Dictionary <string, TopicMetadata>
{
{ "topic1", topic1Metadata },
{ "topic2", topic2Metadata }
};
var intPartitioner = new IntPartitioner();
var config = new ProducerConfig();
config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props);
var producerPool = new ProducerPool(config);
var handler = new DefaultEventHandler <int, string>(
config, intPartitioner, null, new IntEncoder(), producerPool, topicPartitionInfos);
var topic1Broker1Data = new List <KeyedMessage <int, Message> >
{
new KeyedMessage <int, Message>(
"topic1",
0,
new Message(
Encoding.UTF8.GetBytes("msg1"))),
new KeyedMessage <int, Message>(
"topic1",
2,
new Message(
Encoding.UTF8.GetBytes("msg3")))
};
var topic1Broker2Data = new List <KeyedMessage <int, Message> >
{
new KeyedMessage <int, Message>(
"topic1",
-101,
3,
new Message(
Encoding.UTF8.GetBytes("msg4")))
};
var topic2Broker1Data = new List <KeyedMessage <int, Message> >
{
new KeyedMessage <int, Message>(
"topic2",
4,
new Message(
Encoding.UTF8.GetBytes("msg5")))
};
var topic2Broker2Data = new List <KeyedMessage <int, Message> >
{
new KeyedMessage <int, Message>(
"topic2",
-99,
1,
new Message(
Encoding.UTF8.GetBytes("msg2")))
};
var expectedResult = new Dictionary <int, Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > > >
{
{ 0,
new Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > >
{
{ new TopicAndPartition("topic1", 0), topic1Broker1Data },
{ new TopicAndPartition("topic2", 0), topic2Broker1Data }
} },
{ 1,
new Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > >
{
{ new TopicAndPartition("topic1", 1), topic1Broker2Data },
{ new TopicAndPartition("topic2", 1), topic2Broker2Data }
} },
};
var actualResut = handler.PartitionAndCollate(producerDataList);
Assert.Equal(expectedResult.Count, actualResut.Count);
Assert.True(expectedResult.Keys.SequenceEqual(actualResut.Keys));
foreach (var key in expectedResult.Keys)
{
var exptectedInnerDict = expectedResult[key];
var actualInnerDict = actualResut[key];
Assert.Equal(exptectedInnerDict.Count, actualInnerDict.Count);
foreach (var topicAndPartition in exptectedInnerDict.Keys)
{
var exptectedKeyedMsgs = exptectedInnerDict[topicAndPartition];
var actualKeyedMsgs = actualInnerDict[topicAndPartition];
Assert.True(exptectedKeyedMsgs.SequenceEqual(actualKeyedMsgs));
}
}
}
