public void WatermarkOffsets(string bootstrapServers)
{
LogToFile("start WatermarkOffsets");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var testString = "hello world";
DeliveryResult <Null, string> dr;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString
}).Result;
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary.
}
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new List <TopicPartitionOffset>()
{
dr.TopicPartitionOffset
});
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition);
Assert.Equal(getOffsets.Low, Offset.Unset);
// the offset of the next message to be read.
Assert.Equal(getOffsets.High, dr.Offset + 1);
var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20));
Assert.NotEqual(queryOffsets.Low, Offset.Unset);
Assert.Equal(getOffsets.High, queryOffsets.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end WatermarkOffsets");
}
private Metadata.IResponse HandleMetadataRequest(Metadata.IRequest req)
{
switch (req)
{
case Metadata.ListTopics _:
return(new Metadata.Topics(Try <List <TopicMetadata> >
.From(() =>
{
using (var adminClient = new DependentAdminClientBuilder(_consumer.Handle).Build())
{
return adminClient.GetMetadata(_settings.MetadataRequestTimeout).Topics;
}
})));
default:
throw new InvalidOperationException($"Unknown metadata request: {req}");
}
}
public void AddBrokers(string bootstrapServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = "localhost:65533"
};
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
try
{
var metadata = adminClient.GetMetadata(TimeSpan.FromSeconds(3));
Assert.True(false, "Broker should not be reached here");
}
catch (KafkaException e)
{
Assert.Equal(ErrorCode.Local_Transport, e.Error.Code);
}
// test is > 0 note == 1 since bootstrapServers could include more than one broker.
int brokersAdded = adminClient.AddBrokers(bootstrapServers);
Assert.True(brokersAdded > 0, "Should have added one broker or more");
brokersAdded = adminClient.AddBrokers(bootstrapServers);
Assert.True(brokersAdded > 0, "Should have added one broker or more (duplicates considered added)");
var newMetadata = adminClient.GetMetadata(TimeSpan.FromSeconds(3));
Assert.True(newMetadata.Brokers.Count >= 1);
brokersAdded = adminClient.AddBrokers("");
Assert.True(brokersAdded == 0, "Should not have added brokers");
newMetadata = adminClient.GetMetadata(TimeSpan.FromSeconds(3));
Assert.True(newMetadata.Brokers.Count > 0);
}
}
public void Producer_Handles(string bootstrapServers)
{
LogToFile("start Producer_Handles");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
using (var producer1 = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
using (var producer2 = new DependentProducerBuilder <string, string>(producer1.Handle).Build())
using (var producer3 = new DependentProducerBuilder <byte[], byte[]>(producer1.Handle).Build())
using (var producer4 = new DependentProducerBuilder <int, string>(producer2.Handle).Build())
using (var producer5 = new DependentProducerBuilder <int, int>(producer3.Handle).Build())
using (var producer6 = new DependentProducerBuilder <string, byte[]>(producer4.Handle).Build())
using (var producer7 = new ProducerBuilder <double, double>(producerConfig).Build())
using (var adminClient = new DependentAdminClientBuilder(producer7.Handle).Build())
{
var r1 = producer1.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 42 }, Value = new byte[] { 33 }
}).Result;
Assert.Equal(new byte[] { 42 }, r1.Key);
Assert.Equal(new byte[] { 33 }, r1.Value);
Assert.Equal(0, r1.Offset);
var r2 = producer2.ProduceAsync(topic.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Result;
Assert.Equal("hello", r2.Key);
Assert.Equal("world", r2.Value);
var r3 = producer3.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 40 }, Value = new byte[] { 31 }
}).Result;
Assert.Equal(new byte[] { 40 }, r3.Key);
Assert.Equal(new byte[] { 31 }, r3.Value);
var r4 = producer4.ProduceAsync(topic.Name, new Message <int, string> {
Key = 42, Value = "mellow world"
}).Result;
Assert.Equal(42, r4.Key);
Assert.Equal("mellow world", r4.Value);
var r5 = producer5.ProduceAsync(topic.Name, new Message <int, int> {
Key = int.MaxValue, Value = int.MinValue
}).Result;
Assert.Equal(int.MaxValue, r5.Key);
Assert.Equal(int.MinValue, r5.Value);
var r6 = producer6.ProduceAsync(topic.Name, new Message <string, byte[]> {
Key = "yellow mould", Value = new byte[] { 69 }
}).Result;
Assert.Equal("yellow mould", r6.Key);
Assert.Equal(new byte[] { 69 }, r6.Value);
var r7 = producer7.ProduceAsync(topic.Name, new Message <double, double> {
Key = 44.0, Value = 234.4
}).Result;
Assert.Equal(44.0, r7.Key);
Assert.Equal(234.4, r7.Value);
var topicMetadata = adminClient.GetMetadata(singlePartitionTopic, TimeSpan.FromSeconds(10));
Assert.Single(topicMetadata.Topics);
// implicitly check this does not throw.
}
var consumerConfig = new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString()
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
var r1 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 42 }, r1.Key);
Assert.Equal(new byte[] { 33 }, r1.Value);
Assert.Equal(0, r1.Offset);
}
using (var consumer = new ConsumerBuilder <string, string>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 1));
var r2 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("hello", r2.Key);
Assert.Equal("world", r2.Value);
Assert.Equal(1, r2.Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 2));
var r3 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 40 }, r3.Key);
Assert.Equal(new byte[] { 31 }, r3.Value);
Assert.Equal(2, r3.Offset);
}
using (var consumer = new ConsumerBuilder <int, string>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 3));
var r4 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(42, r4.Key);
Assert.Equal("mellow world", r4.Value);
Assert.Equal(3, r4.Offset);
}
using (var consumer = new ConsumerBuilder <int, int>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 4));
var r5 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(int.MaxValue, r5.Key);
Assert.Equal(int.MinValue, r5.Value);
Assert.Equal(4, r5.Offset);
}
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 5));
var r6 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("yellow mould", r6.Key);
Assert.Equal(new byte[] { 69 }, r6.Value);
Assert.Equal(5, r6.Offset);
}
using (var consumer = new ConsumerBuilder <double, double>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 6));
var r7 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(44.0, r7.Key);
Assert.Equal(234.4, r7.Value);
Assert.Equal(6, r7.Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Producer_Handles");
}
public async void AdminClient_AclOperations(string bootstrapServers)
{
LogToFile("start AdminClient_AclOperations");
var topicName = Guid.NewGuid().ToString();
var groupName = Guid.NewGuid().ToString();
var maxDuration = TimeSpan.FromSeconds(30);
var noError = new Error(ErrorCode.NoError, "", false);
var unknownError = new Error(ErrorCode.Unknown, "Unknown broker error", false);
var noErrorCreateResult = new CreateAclReport
{
Error = noError
};
var unknownErrorCreateResult = new CreateAclReport
{
Error = unknownError
};
var newACLs = new List <AclBinding>
{
new AclBinding()
{
Pattern = new ResourcePattern
{
Type = ResourceType.Topic,
Name = topicName,
ResourcePatternType = ResourcePatternType.Literal
},
Entry = new AccessControlEntry
{
Principal = "User:test-user-1",
Host = "*",
Operation = AclOperation.Read,
PermissionType = AclPermissionType.Allow
}
},
new AclBinding()
{
Pattern = new ResourcePattern
{
Type = ResourceType.Topic,
Name = topicName,
ResourcePatternType = ResourcePatternType.Prefixed
},
Entry = new AccessControlEntry
{
Principal = "User:test-user-2",
Host = "*",
Operation = AclOperation.Write,
PermissionType = AclPermissionType.Deny
}
},
new AclBinding()
{
Pattern = new ResourcePattern
{
Type = ResourceType.Group,
Name = groupName,
ResourcePatternType = ResourcePatternType.Prefixed
},
Entry = new AccessControlEntry
{
Principal = "User:test-user-2",
Host = "some-host",
Operation = AclOperation.All,
PermissionType = AclPermissionType.Allow
}
},
};
var invalidACLs = new List <AclBinding>
{
new AclBinding()
{
Pattern = new ResourcePattern
{
Type = ResourceType.Topic,
Name = topicName,
ResourcePatternType = ResourcePatternType.Literal
},
Entry = new AccessControlEntry
{
// Principal must be in the form "{principalType}:{principalName}"
// Broker returns ErrUnknown in this case
Principal = "wrong-principal",
Host = "*",
Operation = AclOperation.Read,
PermissionType = AclPermissionType.Allow
}
}
};
var validAndInvalidACLs = new List <AclBinding>
{
new AclBinding()
{
Pattern = new ResourcePattern
{
Type = ResourceType.Topic,
Name = topicName,
ResourcePatternType = ResourcePatternType.Literal
},
Entry = new AccessControlEntry
{
// Principal must be in the form "{principalType}:{principalName}"
// Broker returns ErrUnknown in this case
Principal = "wrong-principal",
Host = "*",
Operation = AclOperation.Read,
PermissionType = AclPermissionType.Allow
}
},
new AclBinding()
{
Pattern = new ResourcePattern
{
Type = ResourceType.Group,
Name = groupName,
ResourcePatternType = ResourcePatternType.Prefixed
},
Entry = new AccessControlEntry
{
Principal = "User:test-user-2",
Host = "some-host",
Operation = AclOperation.All,
PermissionType = AclPermissionType.Allow
}
},
};
var aclBindingFilters = new List <AclBindingFilter>
{
new AclBindingFilter()
{
PatternFilter = new ResourcePatternFilter
{
Type = ResourceType.Any,
ResourcePatternType = ResourcePatternType.Any
},
EntryFilter = new AccessControlEntryFilter
{
Operation = AclOperation.Any,
PermissionType = AclPermissionType.Any
}
},
new AclBindingFilter()
{
PatternFilter = new ResourcePatternFilter
{
Type = ResourceType.Any,
ResourcePatternType = ResourcePatternType.Prefixed
},
EntryFilter = new AccessControlEntryFilter
{
Operation = AclOperation.Any,
PermissionType = AclPermissionType.Any
}
},
new AclBindingFilter()
{
PatternFilter = new ResourcePatternFilter
{
Type = ResourceType.Topic,
ResourcePatternType = ResourcePatternType.Any
},
EntryFilter = new AccessControlEntryFilter
{
Operation = AclOperation.Any,
PermissionType = AclPermissionType.Any
}
},
new AclBindingFilter()
{
PatternFilter = new ResourcePatternFilter
{
Type = ResourceType.Group,
ResourcePatternType = ResourcePatternType.Any
},
EntryFilter = new AccessControlEntryFilter
{
Operation = AclOperation.Any,
PermissionType = AclPermissionType.Any
}
}
};
var createAclsOptions = new CreateAclsOptions
{
RequestTimeout = maxDuration
};
var describeAclsOptions = new DescribeAclsOptions
{
RequestTimeout = maxDuration
};
var deleteAclsOptions = new DeleteAclsOptions
{
RequestTimeout = maxDuration
};
// - construction of admin client from configuration.
// - creation of multiple ACL.
// - CreateAcls should be idempotent
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build())
{
for (int i = 0; i < 2; ++i)
{
await adminClient.CreateAclsAsync(
newACLs,
createAclsOptions
);
}
}
// - construction of admin client from a producer handle
// - CreateACLs with server side validation errors
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
var createAclsException = await Assert.ThrowsAsync <CreateAclsException>(() =>
adminClient.CreateAclsAsync(
invalidACLs,
createAclsOptions
)
);
Assert.Equal(new CreateAclsException(
new List <CreateAclReport> {
unknownErrorCreateResult
}
), createAclsException);
}
// - construction of admin client from a producer handle
// - CreateACLs with errors and succeeded items
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
var createAclsException = await Assert.ThrowsAsync <CreateAclsException>(() =>
adminClient.CreateAclsAsync(
validAndInvalidACLs,
createAclsOptions
)
);
Assert.Equal(new CreateAclsException(
new List <CreateAclReport> {
unknownErrorCreateResult, noErrorCreateResult
}
), createAclsException);
// DescribeAcls must return the three ACLs
var describeAclsResult = await adminClient.DescribeAclsAsync(aclBindingFilters[0], describeAclsOptions);
var newACLsShuffled = new List <AclBinding>
{
newACLs[0],
newACLs[2],
newACLs[1],
};
Assert.Equal(new DescribeAclsResult
{
AclBindings = newACLs
}, describeAclsResult);
Assert.Equal(new DescribeAclsResult
{
AclBindings = newACLsShuffled
}, describeAclsResult);
}
// - construction of admin client from configuration.
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build())
{
// DeleteAcls with ResourcePatternType Prefixed
var resultDeleteAcls = await adminClient.DeleteAclsAsync(
new List <AclBindingFilter>
{
aclBindingFilters[1]
},
deleteAclsOptions
);
Assert.Single(resultDeleteAcls);
Assert.Equal(2, resultDeleteAcls[0].AclBindings.Count);
Assert.Equal(new DeleteAclsResult
{
AclBindings = new List <AclBinding>
{
newACLs[1],
newACLs[2]
}
}, resultDeleteAcls[0]);
// DeleteAcls with ResourceType Topic and Group
resultDeleteAcls = await adminClient.DeleteAclsAsync(
new List <AclBindingFilter>
{
aclBindingFilters[2],
aclBindingFilters[3],
},
deleteAclsOptions
);
Assert.Equal(2, resultDeleteAcls.Count);
Assert.Equal(new DeleteAclsResult
{
AclBindings = new List <AclBinding>
{
newACLs[0]
}
}, resultDeleteAcls[0]);
Assert.Equal(new DeleteAclsResult
{
AclBindings = new List <AclBinding>()
}, resultDeleteAcls[1]);
// All the ACLs should have been deleted
var describeAclsResult = await adminClient.DescribeAclsAsync(aclBindingFilters[0], describeAclsOptions);
Assert.Equal(new DescribeAclsResult
{
AclBindings = new List <AclBinding> {
}
}, describeAclsResult);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AdminClient_AclOperations");
}
public void AdminClient_CreateTopics(string bootstrapServers)
{
LogToFile("start AdminClient_CreateTopics");
var topicName1 = Guid.NewGuid().ToString();
var topicName2 = Guid.NewGuid().ToString();
var topicName3 = Guid.NewGuid().ToString();
var topicName4 = Guid.NewGuid().ToString();
var topicName5 = Guid.NewGuid().ToString();
// test
// - construction of admin client from configuration.
// - creation of more than one topic.
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build())
{
adminClient.CreateTopicsAsync(
new TopicSpecification[]
{
new TopicSpecification {
Name = topicName1, NumPartitions = 2, ReplicationFactor = 1
},
new TopicSpecification {
Name = topicName2, NumPartitions = 12, ReplicationFactor = 1
}
}
).Wait();
}
// test
// - construction of admin client from a producer handle
// - creation of topic
// - producing to created topics works.
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient2 = new DependentAdminClientBuilder(producer.Handle).Build())
{
adminClient2.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topicName3, NumPartitions = 24, ReplicationFactor = 1
}
}).Wait();
var deliveryReport1 = producer.ProduceAsync(topicName1, new Message <Null, Null>()).Result;
var deliveryReport2 = producer.ProduceAsync(topicName2, new Message <Null, Null>()).Result;
var deliveryReport3 = producer.ProduceAsync(topicName3, new Message <Null, Null>()).Result;
Assert.Equal(topicName1, deliveryReport1.Topic);
Assert.Equal(topicName2, deliveryReport2.Topic);
Assert.Equal(topicName3, deliveryReport3.Topic);
}
// test
// - create topic with same name as existing topic
// - as well as another topic that does exist (and for which create should succeed).
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build())
{
try
{
adminClient.CreateTopicsAsync(new List <TopicSpecification>
{
new TopicSpecification {
Name = topicName3, NumPartitions = 1, ReplicationFactor = 1
},
new TopicSpecification {
Name = topicName4, NumPartitions = 1, ReplicationFactor = 1
}
}
).Wait();
Assert.True(false, "Expect CreateTopics request to throw an exception.");
}
// if awaited, the CreateTopicsException is not wrapped.
// this is an annoyance if used synchronously, but not atypical.
catch (AggregateException ex)
{
Assert.True(ex.InnerException.GetType() == typeof(CreateTopicsException));
var cte = (CreateTopicsException)ex.InnerException;
Assert.Equal(2, cte.Results.Count);
Assert.Single(cte.Results.Where(r => r.Error.IsError));
Assert.Single(cte.Results.Where(r => !r.Error.IsError));
Assert.Equal(topicName3, cte.Results.Where(r => r.Error.IsError).First().Topic);
Assert.Equal(topicName4, cte.Results.Where(r => !r.Error.IsError).First().Topic);
}
}
// test
// - validate only
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build())
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topicName5, NumPartitions = 1, ReplicationFactor = 1
}
},
new CreateTopicsOptions {
ValidateOnly = true, RequestTimeout = TimeSpan.FromSeconds(30)
}
).Wait();
// creating for real shouldn't throw exception.
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topicName5, NumPartitions = 1, ReplicationFactor = 1
}
}
).Wait();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AdminClient_CreateTopics");
}
public void Start()
{
var logContext = $"{nameof(Start)} for SubscriberId={_subscriberId}";
try
{
IConsumer <string, string> consumer = new ConsumerBuilder <string, string>(_consumerProperties).Build();
var processor = new KafkaMessageProcessor(_subscriberId, _handler,
_loggerFactory.CreateLogger <KafkaMessageProcessor>());
using (IAdminClient adminClient = new DependentAdminClientBuilder(consumer.Handle).Build())
{
foreach (string topic in _topics)
{
VerifyTopicExistsBeforeSubscribing(adminClient, topic);
}
}
List <string> topicsList = new List <string>(_topics);
_logger.LogDebug($"{logContext}: Subscribing to topics='{String.Join(",", topicsList)}'");
consumer.Subscribe(topicsList);
// Set state to started before starting the processing thread instead of after as in the Java code
// (prevent setting it to it started after it has potentially already been set to stopped)
_state = EventuateKafkaConsumerState.Started;
Task.Run(() =>
{
try
{
while (!_cancellationTokenSource.IsCancellationRequested)
{
try
{
ConsumeResult <string, string> record =
consumer.Consume(TimeSpan.FromMilliseconds(ConsumePollMilliseconds));
if (record != null)
{
_logger.LogDebug(
$"{logContext}: process record at offset='{record.Offset}', key='{record.Key}', value='{record.Value}'");
processor.Process(record);
}
else
{
processor.ThrowExceptionIfHandlerFailed();
}
MaybeCommitOffsets(consumer, processor);
}
catch (ConsumeException e)
{
_logger.LogError($"{logContext}: ConsumeException - {e.Error}. Continuing.");
}
}
_state = EventuateKafkaConsumerState.Stopped;
}
catch (TaskCanceledException)
{
_logger.LogInformation($"{logContext}: Shutdown by cancel");
_state = EventuateKafkaConsumerState.Stopped;
}
catch (KafkaMessageProcessorFailedException e)
{
_logger.LogError($"{logContext}: Terminating due to KafkaMessageProcessorFailedException - {e}");
_state = EventuateKafkaConsumerState.MessageHandlingFailed;
}
catch (Exception e)
{
_logger.LogError($"{logContext}: Exception - {e}");
_state = EventuateKafkaConsumerState.Failed;
// Java throws here, but seems like it isn't necessary
}
finally
{
// Try to put the last of the offsets away. Note that the
// callbacks are done asynchronously so there is no guarantee
// that all the offsets are ready. Worst case is that there
// are messages processed more than once.
MaybeCommitOffsets(consumer, processor);
consumer.Dispose();
_logger.LogDebug($"{logContext}: Stopped in state {_state.ToString()}");
}
}, _cancellationTokenSource.Token);
}
catch (Exception e)
{
_logger.LogError(e, $"{logContext}: Error subscribing");
_state = EventuateKafkaConsumerState.FailedToStart;
throw;
}
}
public void AdminClient_CreatePartitions(string bootstrapServers)
{
LogToFile("start AdminClient_CreatePartitions");
var topicName1 = Guid.NewGuid().ToString();
var topicName2 = Guid.NewGuid().ToString();
var topicName3 = Guid.NewGuid().ToString();
var topicName4 = Guid.NewGuid().ToString();
var topicName5 = Guid.NewGuid().ToString();
var topicName6 = Guid.NewGuid().ToString();
// test creating a new partition works.
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = topicName1, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
adminClient.CreatePartitionsAsync(new List <PartitionsSpecification> {
new PartitionsSpecification {
Topic = topicName1, IncreaseTo = 2
}
}).Wait();
var dr1 = producer.ProduceAsync(new TopicPartition(topicName1, 0), new Message <Null, Null>()).Result;
var dr2 = producer.ProduceAsync(new TopicPartition(topicName1, 1), new Message <Null, Null>()).Result;
try
{
producer.ProduceAsync(new TopicPartition(topicName1, 2), new Message <Null, Null>()).Wait();
Assert.True(false, "expecting exception");
}
catch (AggregateException ex)
{
Assert.IsType <ProduceException <Null, Null> >(ex.InnerException);
Assert.True(((ProduceException <Null, Null>)ex.InnerException).Error.IsError);
}
}
// check validate only works.
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = topicName2, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
adminClient.CreatePartitionsAsync(new List <PartitionsSpecification> {
new PartitionsSpecification {
Topic = topicName2, IncreaseTo = 10
}
}, new CreatePartitionsOptions {
ValidateOnly = true
}).Wait();
// forces a metadata request.
var dr1 = producer.ProduceAsync(new TopicPartition(topicName2, 0), new Message <Null, Null>()).Result;
try
{
// since we have metadata, this throws immediately (i.e. not wrapped in AggregateException)
var dr2 = producer.ProduceAsync(new TopicPartition(topicName2, 1), new Message <Null, Null>()).Result;
Assert.True(false, "expecting exception");
}
catch (AggregateException ex)
{
Assert.IsType <ProduceException <Null, Null> >(ex.InnerException);
Assert.True(((ProduceException <Null, Null>)ex.InnerException).Error.IsError);
}
}
// check valid Assignments property value works.
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = topicName3, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
adminClient.CreatePartitionsAsync(
new List <PartitionsSpecification>
{
new PartitionsSpecification {
Topic = topicName2, IncreaseTo = 2, ReplicaAssignments = new List <List <int> > {
new List <int> {
0
}
}
}
},
new CreatePartitionsOptions {
ValidateOnly = true
}
).Wait();
}
// check invalid Assignments property value works.
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = topicName4, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
try
{
adminClient.CreatePartitionsAsync(
new List <PartitionsSpecification>
{
new PartitionsSpecification {
Topic = topicName2, IncreaseTo = 2, ReplicaAssignments = new List <List <int> > {
new List <int> {
42
}
}
}
},
new CreatePartitionsOptions {
ValidateOnly = true
}
).Wait();
Assert.True(false, "Expecting exception");
}
catch (AggregateException ex)
{
Assert.True(ex.InnerException.GetType() == typeof(CreatePartitionsException));
var cpe = (CreatePartitionsException)ex.InnerException;
Assert.Single(cpe.Results);
Assert.True(cpe.Results.First().Error.IsError);
}
}
// more than one.
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build())
{
adminClient.CreateTopicsAsync(new TopicSpecification[]
{
new TopicSpecification {
Name = topicName5, NumPartitions = 1, ReplicationFactor = 1
},
new TopicSpecification {
Name = topicName6, NumPartitions = 1, ReplicationFactor = 1
}
}
).Wait();
Thread.Sleep(TimeSpan.FromSeconds(1));
// just a simple check there wasn't an exception.
adminClient.CreatePartitionsAsync(
new List <PartitionsSpecification>
{
new PartitionsSpecification {
Topic = topicName5, IncreaseTo = 2
},
new PartitionsSpecification {
Topic = topicName6, IncreaseTo = 3
}
}
).Wait();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AdminClient_CreatePartitions");
}
public void AdminClient_NullReferenceChecks(string bootstrapServers)
{
LogToFile("start AdminClient_NullReferenceChecks");
var topicName1 = Guid.NewGuid().ToString();
string nullTopic = null;
Exception createTopicsException = null;
Exception createPartitionsException = null;
// test creating a null topic throws a related exception
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
try
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = nullTopic, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
Assert.True(false, "Expected exception.");
}
catch (ArgumentException ex)
{
Assert.Contains("topic", ex.Message.ToLower());
createTopicsException = ex;
}
}
// test creating a partition with null topic throws exception
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
try
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = topicName1, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
adminClient.CreatePartitionsAsync(new List <PartitionsSpecification> {
new PartitionsSpecification {
Topic = nullTopic, IncreaseTo = 2
}
}).Wait();
Assert.True(false, "Expected exception.");
}
catch (ArgumentException ex)
{
Assert.Contains("topic", ex.Message.ToLower());
createPartitionsException = ex;
}
}
Assert.True(createTopicsException != null && createPartitionsException != null);
Assert.True(createTopicsException.GetType() == createPartitionsException.GetType(), ".CreateTopic and .CreatePartition should have consistent interface for null-related exceptions.");
// test adding a null list of brokers throws null reference exception.
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
try
{
adminClient.AddBrokers(null);
Assert.True(false, "Expected exception.");
}
catch (ArgumentNullException ex)
{
Assert.Contains("broker", ex.Message.ToLower());
}
}
// test retrieving metadata for a null topic
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
try
{
adminClient.GetMetadata(null, TimeSpan.FromSeconds(10));
Assert.True(false, "Expected exception.");
}
catch (ArgumentNullException ex)
{
Assert.Contains("value cannot be null", ex.Message.ToLower());
}
}
// Deleting null topic throws exception
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
try
{
adminClient.DeleteTopicsAsync(new List <string> {
topicName1, nullTopic
});
Assert.True(false, "Expected exception.");
}
catch (ArgumentException ex)
{
Assert.Contains("topic", ex.Message);
}
}
// ListGroup throws exception if group is null
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build())
{
try
{
adminClient.ListGroup(null, TimeSpan.FromSeconds(10));
Assert.True(false, "Expected exception.");
}
catch (ArgumentNullException ex)
{
Assert.Contains("group", ex.Message);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AdminClient_NullReferenceChecks");
}
