public void Awake(){ // Populate the partition interactable parent dictionary, only for ones existing foreach(Transform trans in partitionParent){ PartitionMetadata metadata = trans.GetComponent<PartitionMetadata>(); if(metadata != null){ partitionInteractableDictionary.Add(metadata.partitionNumber, metadata.interactables); } else{ Debug.LogError("Non partition detected " + trans.name); } } }
public void Constuctor() { int[] replicas = new int[0]; int[] inSyncReplicas = new int[0]; var pm = new PartitionMetadata(33, 1, replicas, inSyncReplicas, ErrorCode.Local_AllBrokersDown); Assert.Equal(pm.PartitionId, 33); Assert.Equal(pm.Leader, 1); Assert.Same(pm.Replicas, replicas); Assert.Same(pm.InSyncReplicas, inSyncReplicas); Assert.Equal(pm.Error, new Error(ErrorCode.Local_AllBrokersDown)); }
public void Constuctor() { int[] replicas = new int[0]; int[] inSyncReplicas = new int[0]; var pm = new PartitionMetadata(33, 1, replicas, inSyncReplicas, ErrorCode._ALL_BROKERS_DOWN); Assert.Equal(pm.PartitionId, 33); Assert.Equal(pm.Leader, 1); Assert.Same(pm.Replicas, replicas); Assert.Same(pm.InSyncReplicas, inSyncReplicas); Assert.Equal(pm.Error, new Error(ErrorCode._ALL_BROKERS_DOWN)); }
public int Parse(byte[] data, int dataOffset) { int bufferOffset = dataOffset; bufferOffset = BufferReader.Read(data, bufferOffset, out topicErrorCode); bufferOffset = BufferReader.Read(data, bufferOffset, out topicName); PartitionMetadatas = new Dictionary<int, PartitionMetadata>(); int partitionCount; bufferOffset = BufferReader.Read(data, bufferOffset, out partitionCount); for (var i = 0; i < partitionCount; i++) { PartitionMetadata metadata = new PartitionMetadata(); bufferOffset = metadata.Parse(data, bufferOffset); PartitionMetadatas.Add(metadata.PartitionId, metadata); } return bufferOffset; }
public async Task CanLoadWatermarksWithValidParamsAsync() { // Arrange var topic = new TopicName("test"); var clientMock = new Mock <IAdminClient>(); var client = clientMock.Object; var timeout = 1000; var loader = new TopicWatermarkLoader(topic, client, timeout); var consumerMock = new Mock <IConsumer <object, object> >(); IConsumer <object, object> consumerFactory() => consumerMock.Object; var adminClientPartition = new TopicPartition(topic.Value, new Partition(1)); var adminParitions = new[] { adminClientPartition }; var borkerMeta = new BrokerMetadata(1, "testHost", 1000); var partitionMeta = new PartitionMetadata(1, 1, new[] { 1 }, new[] { 1 }, null); var topicMeta = new TopicMetadata(topic.Value, new[] { partitionMeta }.ToList(), null); var meta = new Confluent.Kafka.Metadata( new[] { borkerMeta }.ToList(), new[] { topicMeta }.ToList(), 1, "test" ); clientMock.Setup(c => c.GetMetadata(topic.Value, TimeSpan.FromSeconds(timeout))).Returns(meta); var offets = new WatermarkOffsets(new Offset(1), new Offset(2)); consumerMock.Setup(x => x.QueryWatermarkOffsets(adminClientPartition, TimeSpan.FromSeconds(timeout))).Returns(offets); TopicWatermark result = null !; // Act var exception = await Record.ExceptionAsync(async() => result = await loader.LoadWatermarksAsync(consumerFactory, CancellationToken.None)); // Assert exception.Should().BeNull(); consumerMock.Verify(x => x.Close(), Times.Once); consumerMock.Verify(x => x.Dispose(), Times.Once); result.Should().NotBeNull(); var watermarks = result.Watermarks.ToList(); watermarks.Should().ContainSingle(); clientMock.Verify(c => c.GetMetadata(topic.Value, TimeSpan.FromSeconds(timeout)), Times.Once); consumerMock.Verify(x => x.QueryWatermarkOffsets(adminClientPartition, TimeSpan.FromSeconds(timeout)), Times.Once); watermarks.Single().TopicName.Should().Be(topic); watermarks.Single().Partition.Value.Should().Be(partitionMeta.PartitionId); watermarks.Single().Offset.Should().Be(offets); }
public int InsertPartitionMetadata(int exhibitId, int mediaId) { return(PartitionMetadata.InsertPartitionMetadata(exhibitId, 1, mediaId, "", "IntegrationTest", FileSystemType.FAT)); }
private async Task Fetch(ChannelWriter <Message <TKey, TValue> > messageWriter, string topic, PartitionMetadata partition, BrokerConnection broker, CancellationToken cancellationToken) { long offset = 0; while (!cancellationToken.IsCancellationRequested) { var fetch = await broker.Fetch(new FetchRequest { ReplicaId = -1, MaxWaitTime = 100, MinBytes = 1, MaxBytes = 64 * 1024, IsolationLevel = 0, SessionId = 0, SessionEpoc = -1, Topics = new [] { new FetchRequest.Topic { TopicName = topic, Partitions = new [] { new FetchRequest.Topic.Partition { PartitionId = partition.PartitionId, CurrentLeaderEpoc = -1, FetchOffset = offset, LogStartOffset = -1, PartitionMaxBytes = 32 * 1024 } } } } }); long maxoffset = 0; long high = 0; foreach (var r in fetch.Responses) { foreach (var pr in r.PartitionResponses) { high = pr.HighWaterMark; foreach (var batch in pr.RecordBatches) { foreach (var rec in batch.Records) { if (rec.Offset > maxoffset) { maxoffset = rec.Offset; } var msg = new Message <TKey, TValue> { Timestamp = Timestamp.UnixTimestampMsToDateTime(batch.FirstTimeStamp + rec.TimeStampDelta), Topic = r.TopicName, PartitionId = pr.PartitionId, Offset = rec.Offset, Key = await keyDeserializer.Deserialize(rec.Key), Value = await valueDeserializer.Deserialize(rec.Value) }; if (rec.Headers.Count > 0) { foreach (var h in rec.Headers) { msg.AddHeader(h.Key, h.Value); } } await messageWriter.WriteAsync(msg, cancellationToken); } } } } if (maxoffset > 0) { offset = maxoffset + 1; } if (offset >= high) { await Task.Delay(200); // linger } } }
private async Task Fetch(ChannelWriter <RecordBatch.Record> messageWriter, string topic, PartitionMetadata partition, BrokerConnection broker, CancellationToken cancellationToken) { long offset = 0; while (!cancellationToken.IsCancellationRequested) { var fetch = await broker.Fetch(new FetchRequest { ReplicaId = -1, MaxWaitTime = 100, MinBytes = 1, MaxBytes = 64 * 1024, IsolationLevel = 0, SessionId = 0, SessionEpoc = -1, Topics = new [] { new FetchRequest.Topic { TopicName = topic, Partitions = new [] { new FetchRequest.Topic.Partition { PartitionId = partition.PartitionId, CurrentLeaderEpoc = -1, FetchOffset = offset, LogStartOffset = -1, PartitionMaxBytes = 32 * 1024 } } } } }); long maxoffset = 0; long high = 0; foreach (var r in fetch.Responses) { foreach (var pr in r.PartitionResponses) { high = pr.HighWaterMark; foreach (var batch in pr.RecordBatches) { foreach (var rec in batch.Records) { if (rec.Offset > maxoffset) { maxoffset = rec.Offset; } await messageWriter.WriteAsync(rec, cancellationToken); } } } } if (maxoffset > 0) { offset = maxoffset + 1; } if (offset >= high) { await Task.Delay(200); // linger } } }
public void TestPartitionAndCollateEvents() { var producerDataList = new List <KeyedMessage <int, Message> >(); // use bogus key and partition key override for some messages producerDataList.Add(new KeyedMessage <int, Message>("topic1", 0, new Message(Encoding.UTF8.GetBytes("msg1")))); producerDataList.Add(new KeyedMessage <int, Message>("topic2", -99, 1, new Message(Encoding.UTF8.GetBytes("msg2")))); producerDataList.Add(new KeyedMessage <int, Message>("topic1", 2, new Message(Encoding.UTF8.GetBytes("msg3")))); producerDataList.Add(new KeyedMessage <int, Message>("topic1", -101, 3, new Message(Encoding.UTF8.GetBytes("msg4")))); producerDataList.Add(new KeyedMessage <int, Message>("topic2", 4, new Message(Encoding.UTF8.GetBytes("msg5")))); var broker1 = new Broker(0, "localhost", 9092); var broker2 = new Broker(1, "localhost", 9093); // form expected partitions metadata var partition1Metadata = new PartitionMetadata(0, broker1, new List <Broker> { broker1, broker2 }); var partition2Metadata = new PartitionMetadata(1, broker2, new List <Broker> { broker1, broker2 }); var topic1Metadata = new TopicMetadata( "topic1", new List <PartitionMetadata> { partition1Metadata, partition2Metadata }); var topic2Metadata = new TopicMetadata( "topic2", new List <PartitionMetadata> { partition1Metadata, partition2Metadata }); var topicPartitionInfos = new Dictionary <string, TopicMetadata> { { "topic1", topic1Metadata }, { "topic2", topic2Metadata } }; var intPartitioner = new IntPartitioner(); var config = new ProducerConfig(); config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props); var producerPool = new ProducerPool(config); var handler = new DefaultEventHandler <int, string>( config, intPartitioner, null, new IntEncoder(), producerPool, topicPartitionInfos); var topic1Broker1Data = new List <KeyedMessage <int, Message> > { new KeyedMessage <int, Message>( "topic1", 0, new Message( Encoding.UTF8.GetBytes("msg1"))), new KeyedMessage <int, Message>( "topic1", 2, new Message( Encoding.UTF8.GetBytes("msg3"))) }; var topic1Broker2Data = new List <KeyedMessage <int, Message> > { new KeyedMessage <int, Message>( "topic1", -101, 3, new Message( Encoding.UTF8.GetBytes("msg4"))) }; var topic2Broker1Data = new List <KeyedMessage <int, Message> > { new KeyedMessage <int, Message>( "topic2", 4, new Message( Encoding.UTF8.GetBytes("msg5"))) }; var topic2Broker2Data = new List <KeyedMessage <int, Message> > { new KeyedMessage <int, Message>( "topic2", -99, 1, new Message( Encoding.UTF8.GetBytes("msg2"))) }; var expectedResult = new Dictionary <int, Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > > > { { 0, new Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > > { { new TopicAndPartition("topic1", 0), topic1Broker1Data }, { new TopicAndPartition("topic2", 0), topic2Broker1Data } } }, { 1, new Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > > { { new TopicAndPartition("topic1", 1), topic1Broker2Data }, { new TopicAndPartition("topic2", 1), topic2Broker2Data } } }, }; var actualResut = handler.PartitionAndCollate(producerDataList); Assert.Equal(expectedResult.Count, actualResut.Count); Assert.True(expectedResult.Keys.SequenceEqual(actualResut.Keys)); foreach (var key in expectedResult.Keys) { var exptectedInnerDict = expectedResult[key]; var actualInnerDict = actualResut[key]; Assert.Equal(exptectedInnerDict.Count, actualInnerDict.Count); foreach (var topicAndPartition in exptectedInnerDict.Keys) { var exptectedKeyedMsgs = exptectedInnerDict[topicAndPartition]; var actualKeyedMsgs = actualInnerDict[topicAndPartition]; Assert.True(exptectedKeyedMsgs.SequenceEqual(actualKeyedMsgs)); } } }