public static string GetTopic(ILogger logger, int numberOfPartitions = 1)
{
string result = null;
WithAdminClient(logger, adminClient =>
{
var topics = adminClient
.GetMetadata(TimeSpan.FromSeconds(10))
.Topics.Select(topic => topic.Topic)
.Select(topic => topic.Split('-'))
.Where(parts => parts.Length == 2 && int.TryParse(parts[1], out _))
.Select(parts => int.Parse(parts[1]))
.ToList();
var number = topics.Any() ? topics.Max() : 0;
var topicName = $"testtopic-{number + 1}";
logger.Information("Using topic named {topic}", topicName);
result = topicName;
var topicSpecification = new TopicSpecification {
Name = topicName, NumPartitions = numberOfPartitions, ReplicationFactor = 1
};
adminClient
.CreateTopicsAsync(new[] { topicSpecification })
.Wait();
});
return(result);
}
protected virtual async Task CreateTopicAsync()
{
using (var adminClient = new AdminClientBuilder(Options.Connections.GetOrDefault(ConnectionName)).Build())
{
var topic = new TopicSpecification
{
Name = TopicName,
NumPartitions = 1,
ReplicationFactor = 1
};
Options.ConfigureTopic?.Invoke(topic);
try
{
await adminClient.CreateTopicsAsync(new[] { topic });
}
catch (CreateTopicsException e)
{
if (e.Results.Any(x => x.Error.Code != ErrorCode.TopicAlreadyExists))
{
throw;
}
}
}
}
/// <summary>
/// Awaitable task to create a single topic by name, if it does not exist yet.
/// </summary>
/// <param name="topic"></param>
/// <param name="numPartitions"></param>
/// <param name="replicationFactor"></param>
/// <param name="configuration"></param>
/// <returns></returns>
public static async Task CreateTopicIfNotExists(string topic, int numPartitions, short replicationFactor, ClientConfig configuration)
{
using (var adminClient = new AdminClientBuilder(configuration).Build())
{
try
{
var topicSpecification = new TopicSpecification
{
Name = topic,
NumPartitions = numPartitions,
ReplicationFactor = replicationFactor
};
// CreateTopicAsync can take a list, but for here only a single topic is passed as input.
await adminClient.CreateTopicsAsync(new List <TopicSpecification> {
topicSpecification
});
Console.WriteLine($"Topic {topic} created.");
}
catch (CreateTopicsException ex)
{
if (ex.Results[0].Error.Code != ErrorCode.TopicAlreadyExists)
{
Console.WriteLine($"An error occured creating topic {topic}: {ex.Results[0].Error.Reason}");
}
else
{
Console.WriteLine($"Topic {topic} already exists.");
}
}
}
}
public IActionResult OnGet(string link)
{
if (link == null)
{
return(NotFound());
}
var resultLink = _repository.Single(LinkSpecification.ByUrl(link));
if (resultLink == null)
{
return(NotFound());
}
var resultTopic = _repository.SingleInclude(BaseSpecification <Topic> .ById(resultLink.TopicId), new List <ISpecification <Topic> > {
TopicSpecification.IncludeQuestions()
});
if (resultTopic == null)
{
return(NotFound());
}
Questions = resultTopic.Questions;
Link = link;
Title = resultTopic.Title;
return(Page());
}
protected virtual async Task CreateTopicAsync()
{
using (var adminClient = new AdminClientBuilder(Options.Connections.GetOrDefault(ConnectionName)).Build())
{
var topic = new TopicSpecification
{
Name = TopicName,
NumPartitions = 1,
ReplicationFactor = 1
};
Options.ConfigureTopic?.Invoke(topic);
try
{
await adminClient.CreateTopicsAsync(new[] { topic });
}
catch (CreateTopicsException e)
{
if (!e.Error.Reason.Contains($"Topic '{TopicName}' already exists"))
{
throw;
}
}
}
}
public bool CreateTopic(TopicSpecification topicSpecification)
{
if (topicSpecification == null)
{
throw new ArgumentNullException(nameof(topicSpecification));
}
var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(30));
if (!metadata.Topics.Exists(x => x.Topic == topicSpecification.Name))
{
try
{
_adminClient.CreateTopicsAsync(new[]
{
topicSpecification
}).GetAwaiter().GetResult();
}
catch (Exception e)
{
//just in case we have a race condition
if (!e.Message.Contains("already exists"))
{
throw;
}
}
}
return(true);
}
public async Task Create(ITopicConfiguration topicConfig)
{
if (!_adminClient.TopicExists(topicConfig.Name, out _))
{
TopicSpecification topicSpecification = new TopicSpecification
{
Name = topicConfig.Name,
NumPartitions = topicConfig.Partitions,
ReplicationFactor = topicConfig.ReplicationFactor,
Configs = new Dictionary <string, string>
{
{ "retention.ms", topicConfig.RetentionMs.ToString() }
}
};
try
{
await _adminClient.CreateTopicsAsync(new[] { topicSpecification });
}
catch (CreateTopicsException ex)
{
Console.WriteLine(ex);
}
_adminClient.EnsureTopicCreation(topicConfig.Name);
}
}
public static async Task AssureTopic(string host, string topic)
{
var adminClientConfig = new AdminClientConfig();
adminClientConfig.BootstrapServers = host;
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
try
{
var metadata = adminClient.GetMetadata(topic, TimeSpan.FromSeconds(10));
if (!metadata.Topics.Any(x => x.Topic == topic))
{
var topicSpecification = new TopicSpecification()
{
Name = topic,
ReplicationFactor = 1,
NumPartitions = 1
};
await adminClient.CreateTopicsAsync(new TopicSpecification[] { topicSpecification });
}
}
catch (Exception ex)
{
throw new Exception($"{nameof(KafkaCommon)} failed to create topic {topic}", ex);
}
}
}
public async Task Create(ITopicConfiguration topicConfig)
{
if (!_adminClient.Value.TopicExists(topicConfig.Name, out _))
{
TopicSpecification topicSpecification = new TopicSpecification
{
Name = topicConfig.Name,
NumPartitions = topicConfig.Partitions,
ReplicationFactor = topicConfig.ReplicationFactor,
Configs = new Dictionary <string, string>
{
{ "retention.ms", topicConfig.RetentionMs.ToString() }
}
};
try
{
await _adminClient.Value.CreateTopicsAsync(new[] { topicSpecification });
_log.Information($"{topicConfig.Name} created.");
}
catch (CreateTopicsException ex)
{
_log.Error(ex, "An unexpected exception occurred when creating topics.");
}
_adminClient.Value.EnsureTopicCreation(topicConfig.Name);
}
}
public async Task CreateTopicAsync(string topic, int?numPartitions, short?replicationFactor,
Dictionary <string, string> config, TimeSpan timeout)
{
var topicSpecification = new TopicSpecification
{
Name = topic,
NumPartitions = numPartitions ?? -1,
ReplicationFactor = replicationFactor ?? -1,
Configs = config
};
var options = new CreateTopicsOptions
{
RequestTimeout = timeout
};
try
{
await _adminClient.CreateTopicsAsync(new[] { topicSpecification }, options);
}
catch (KafkaException e)
{
throw new AdminClientException("Unable to create topic.", e);
}
}
/// <summary>
/// Create all topics used by this example, if they don't already exist.
/// </summary>
static async Task CreateTopicsMaybe(string brokerList, string clientId)
{
var config = new AdminClientConfig
{
BootstrapServers = brokerList,
ClientId = clientId
};
using (var adminClent = new AdminClientBuilder(config).Build())
{
var countsTopicSpec = new TopicSpecification
{
Name = Topic_Counts,
// note: in production, you should generally use a replication factor of 3.
ReplicationFactor = 1,
NumPartitions = NumPartitions,
// this topic backs a kv (word -> count) state store, so it can be compacted.
Configs = new Dictionary <string, string> {
{ "cleanup.policy", "compact" }
}
};
var wordsTopicSpec = new TopicSpecification
{
Name = Topic_Words,
ReplicationFactor = 1,
NumPartitions = NumPartitions
};
var inputLinesTopicSpec = new TopicSpecification
{
Name = Topic_InputLines,
ReplicationFactor = 1,
NumPartitions = NumPartitions
};
try
{
await adminClent.CreateTopicsAsync(new List <TopicSpecification> {
countsTopicSpec, wordsTopicSpec, inputLinesTopicSpec
});
}
catch (CreateTopicsException ex)
{
// propagate the exception unless the error was that one or more of the topics already exists.
if (ex.Results.Select(r => r.Error.Code).Where(el => el != ErrorCode.TopicAlreadyExists && el != ErrorCode.NoError).Count() > 0)
{
throw new Exception("Unable to create topics", ex);
}
}
}
}
public static void AddKafkaProducer <TKey, TValue>(this IServiceCollection services, IConfigurationSection configurationSection)
{
var connectionString = configurationSection.GetValue <string>("ConnectionString");
var producerConfigurationSection = configurationSection.GetSection("Producers");
var producerOptions = new List <ProducerOptions>();
producerConfigurationSection.Bind(producerOptions);
var producer = producerOptions.FirstOrDefault(x =>
string.Equals(x.MessageType, typeof(TValue).Name, StringComparison.InvariantCultureIgnoreCase));
if (producer == null)
{
throw new ArgumentNullException($"No producer configuration for the message type found");
}
var producerConfig = new ProducerConfig(producer.Configurations)
{
BootstrapServers = connectionString
};
if (producer.EnableTopicCreation)
{
var topicSpecification = new TopicSpecification
{
Name = typeof(TValue).Name,
NumPartitions = producer.NumPartitions
};
new KafkaTopicFactory(producerConfig).CreateTopic(topicSpecification);
}
services.AddSingleton(provider =>
{
var logger = provider.GetRequiredService <ILogger <IProducer <TKey, TValue> > >();
var kafkaProducer = new ProducerBuilder <TKey, TValue>(producerConfig)
.SetErrorHandler((x, e) => logger.LogError($"ErrorCode: {e.Code}, Reason: {e.Reason}"))
.SetLogHandler((x, e) => logger.LogInformation($"LogLevel: {e.Level}, LogName: {e.Name}, LogMessage: {e.Message}"))
.SetKeySerializer(new JsonSerializer <TKey>())
.SetValueSerializer(new JsonSerializer <TValue>())
.Build();
return(kafkaProducer);
});
}
private static void ExecuteTopicCreate(ManageOptions options)
{
using (var client = GetAdminClient(options.BootstrapServer))
{
var topic = new TopicSpecification()
{
Name = options.Name, NumPartitions = options.Partitions, ReplicationFactor = options.ReplicationFactor
};
var topics = new List <TopicSpecification>()
{
topic
};
try
{
var topicTask = client.CreateTopicsAsync(topics, new CreateTopicsOptions()
{
RequestTimeout = _requestTimeout, OperationTimeout = _requestTimeout
}) as Task <List <CreateTopicReport> >;
var topicResults = topicTask.Result;
// TODO: Refactor reporting output to generalize it. Colorize based on individual report status.
foreach (var result in topicResults)
{
Console.WriteLine($"Topic '{result.Topic}' creation result: '{result.Error}'.");
}
}
catch (AggregateException ex) when(ex.InnerException is CreateTopicsException)
{
var cte = ex.InnerException as CreateTopicsException;
foreach (var result in cte.Results)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"Topic '{result.Topic}' creation result: '{result.Error}'.");
Console.ResetColor();
}
}
}
}
public async Task <bool> EnsureTopicExists(string topicName)
{
_logger.LogInformation("Ensuring topic exists...");
var topicSpecification = new TopicSpecification()
{
Name = topicName,
NumPartitions = 1,
ReplicationFactor = 1
};
var topicSpecifications = new List <TopicSpecification>();
topicSpecifications.Add(topicSpecification);
// try and get metadata about topic to check that it exists
var metaData = _adminClient.GetMetadata(topicName, TimeSpan.FromSeconds(10));
if (!metaData.Topics[0].Error.IsError)
{
_logger.LogInformation(" Topic already exists.");
return(true);
}
_logger.LogInformation(" Creating topic...");
try
{
await _adminClient.CreateTopicsAsync(topicSpecifications);
_logger.LogInformation(" Topic created.");
}
catch (CreateTopicsException e)
{
// only trap fatal errors so that we let topic already exists error through
if (e.Error.IsFatal)
{
return(false);
}
}
return(true);
}
public async Task AddTopicAsync(Topic topic)
{
try
{
var topicsSpecification = new TopicSpecification[]
{
new TopicSpecification
{
Name = topic.Nome,
NumPartitions = topic.Particoes.GetValueOrDefault(6),
ReplicationFactor = topic.Replicas.GetValueOrDefault(3)
}
};
await client.CreateTopicsAsync(topicsSpecification);
}
catch (CreateTopicsException exception)
{
throw exception;
}
}
protected async Task TryCreateTopic(string topicName, int partitionsCount)
{
try
{
var topicSpec = new TopicSpecification
{
Name = topicName,
ReplicationFactor = 1,
NumPartitions = partitionsCount,
Configs = new Dictionary <string, string>()
};
if (_useCompact)
{
topicSpec.Configs.Add("cleanup.policy", "compact");
}
await AdminClient.CreateTopicsAsync(new TopicSpecification[] { topicSpec });
}
catch (CreateTopicsException e)
{
Console.WriteLine($"{e.Error.Reason}");
}
}
public async Task CreateSingleTopic(AdminClientConfig adminConfig, TopicSpecification topicSpecs)
{
_logger.LogInformation(" Entered CreateSingleTopic() with adminConfig : {0} , TopicSpecification : {1} ", adminConfig, topicSpecs);
using (var adminclient = new AdminClientBuilder(adminConfig).Build())
{
try
{
await adminclient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = "topic", NumPartitions = 2, ReplicationFactor = 1
} });
}
catch (CreateTopicsException e)
{
e.Results.ForEach(r => Console.WriteLine(r.Error.IsError ? "had error" : r.Error.Reason));
_logger.LogError("CreateTopicsException occured at CreateSingleTopic() : {0}", e);
}
catch (Exception ex)
{
_logger.LogError("Exception occured at CreateSingleTopic() : {0}", ex);
}
}
}
private static void CreateTopic()
{
var config = new AdminClientConfig()
{
BootstrapServers = _server
};
var topic = new TopicSpecification()
{
Name = _topicName, NumPartitions = 1, ReplicationFactor = 1
};
var topics = new List <TopicSpecification>()
{
topic
};
using (var client = new AdminClient(config))
{
var topicTask = client.CreateTopicsAsync(topics) as Task <List <CreateTopicExceptionResult> >;
var topicResults = topicTask.Result;
}
}
public static async Task CreateTopic(string host, string topic)
{
var adminClientConfig = new AdminClientConfig();
adminClientConfig.BootstrapServers = host;
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
try
{
var topicSpecification = new TopicSpecification()
{
Name = topic,
ReplicationFactor = 1,
NumPartitions = 1
};
await adminClient.CreateTopicsAsync(new TopicSpecification[] { topicSpecification });
}
catch (Exception ex)
{
throw new Exception($"{nameof(KafkaCommon)} failed to create topic {topic}", ex);
}
}
}
public ConfigureTopologyFilter(IReadOnlyDictionary <string, string> clientConfig, KafkaTopicOptions options)
{
_options = options;
_specification = _options.ToSpecification();
_config = new AdminClientConfig(clientConfig.ToDictionary(x => x.Key, x => x.Value));
}
public IActionResult OnPost()
{
var resultLink = _repository.Single(LinkSpecification.ByUrl(ShortcutLink));
if (resultLink == null)
{
return(NotFound());
}
var resultTopic = _repository.SingleInclude(BaseSpecification <Topic> .ById(resultLink.TopicId), new List <ISpecification <Topic> > {
TopicSpecification.IncludeQuestions()
});
if (resultTopic == null)
{
return(NotFound());
}
if (resultTopic.PreventNSFW)
{
if (IsNsfw(Question.Content))
{
ModelState.AddModelError("Question.Content", "Question contains word/s that are considered as NSFW");
}
}
if (resultTopic.PreventSpam)
{
if (!ReCaptchaHelper.ValidateRecaptcha(Request.Form["g-recaptcha-response"]))
{
ModelState.AddModelError("Question.Content", "Invalid ReCaptcha");
}
}
switch (resultTopic.DuplicationCheck)
{
case DuplicationCheck.IpAddress:
{
if (IpAddressExists(WebHelper.GetRemoteIP, resultTopic.Id))
{
ModelState.AddModelError("Question.Content", "You have already asked a question on this poll");
break;
}
else if (ModelState.IsValid)
{
AddIpAddress(resultTopic.Id);
}
break;
}
case DuplicationCheck.BrowserCookie:
{
const string cookieKey = "PID";
if (CookieExists(cookieKey, ShortcutLink))
{
ModelState.AddModelError("Question.Content", "You have already asked a question on this poll");
break;
}
else if (ModelState.IsValid)
{
WebHelper.SetCookie(cookieKey, ShortcutLink, null);
}
break;
}
case DuplicationCheck.None: break;
default:
_logger.LogError($"Unkown duplication check case {resultTopic.DuplicationCheck}.");
throw new InvalidEnumArgumentException("Unrecognized case value.");
}
if (!ModelState.IsValid)
{
return(Page());
}
resultTopic.Questions.Add(Question);
_repository.Update(resultTopic);
InterrogateClient.UpdateList(ShortcutLink, Question);
InterrogateClient.UpdateQuestionCount(ShortcutLink, resultTopic.Questions.Count);
return(RedirectToPage("Result"));
}
public async Task Should_bypass_if_created()
{
var services = new ServiceCollection();
const ushort partitionCount = 2;
const short replicaCount = BrokersCount;
const string topicName = "do-not-create-topic";
TaskCompletionSource <ConsumeContext <KafkaMessage> > taskCompletionSource = GetTask <ConsumeContext <KafkaMessage> >();
services.TryAddSingleton <ILoggerFactory>(LoggerFactory);
services.TryAddSingleton(typeof(ILogger <>), typeof(Logger <>));
services.AddSingleton(taskCompletionSource);
services.AddMassTransit(x =>
{
x.UsingInMemory((context, cfg) => cfg.ConfigureEndpoints(context));
x.AddRider(rider =>
{
rider.AddConsumer <KafkaMessageConsumer>();
rider.AddProducer <KafkaMessage>(topicName);
rider.UsingKafka((context, k) =>
{
k.Host(Host);
k.TopicEndpoint <KafkaMessage>(topicName, nameof(ConfigureTopology_Specs), c =>
{
c.AutoOffsetReset = AutoOffsetReset.Earliest;
c.ConfigureConsumer <KafkaMessageConsumer>(context);
c.CreateIfMissing(t =>
{
t.NumPartitions = partitionCount;
t.ReplicationFactor = replicaCount;
});
});
});
});
});
var provider = services.BuildServiceProvider();
var busControl = provider.GetRequiredService <IBusControl>();
var config = new AdminClientConfig {
BootstrapServers = Host
};
var client = new AdminClientBuilder(config).Build();
var specification = new TopicSpecification
{
Name = topicName,
NumPartitions = partitionCount,
ReplicationFactor = replicaCount
};
await client.CreateTopicsAsync(new[] { specification });
await busControl.StartAsync(TestCancellationToken);
var serviceScope = provider.CreateScope();
var producer = serviceScope.ServiceProvider.GetRequiredService <ITopicProducer <KafkaMessage> >();
try
{
await producer.Produce(new { Text = "text" }, TestCancellationToken);
ConsumeContext <KafkaMessage> result = await taskCompletionSource.Task;
Assert.NotNull(result);
}
finally
{
await busControl.StopAsync(TestCancellationToken);
await provider.DisposeAsync();
try
{
await client.DeleteTopicsAsync(new[] { topicName });
}
catch (DeleteTopicsException)
{
//suppress
}
finally
{
client.Dispose();
}
}
}
async static Task Main(string[] args)
{
if (args.Length == 0)
{
Console.WriteLine("usage: .. <recreate|fakegen|process|log> <bootstrap servers> [instance id]");
Environment.Exit(1);
}
var command = args[0];
var brokerAddress = args[1];
var instanceId = args.Length == 3 ? args[2] : null;
var simulatedWeblogTopic = "simulated-weblog";
var filteredWeblogTopic = "filtered-weblog";
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
switch (command)
{
case "recreate":
var topicSpecs = new TopicSpecification[]
{
new TopicSpecification {
Name = simulatedWeblogTopic, NumPartitions = 1, ReplicationFactor = 1
},
new TopicSpecification {
Name = filteredWeblogTopic, NumPartitions = 1, ReplicationFactor = 1
}
};
Console.WriteLine("recreating topics...");
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = brokerAddress
}).Build())
{
try
{
await adminClient.DeleteTopicsAsync(topicSpecs.Select(ts => ts.Name));
}
catch (DeleteTopicsException ex)
{
foreach (var deleteResult in ex.Results)
{
if (deleteResult.Error.Code != ErrorCode.UnknownTopicOrPart)
{
throw;
}
}
}
await Task.Delay(TimeSpan.FromSeconds(5));
await adminClient.CreateTopicsAsync(topicSpecs);
}
Console.WriteLine("done.");
break;
case "fakegen":
// 1. A processor that generates some fake weblog data.
Random r = new Random();
var fakeDataSourceProcessor = new Processor <Null, Null, Null, string>
{
BootstrapServers = brokerAddress,
OutputTopic = simulatedWeblogTopic,
Function = (_) =>
{
Thread.Sleep(r.Next(1000));
return(new Message <Null, string> {
Value = WebLogLine.GenerateFake()
});
}
};
fakeDataSourceProcessor.Start(instanceId, cts.Token);
break;
case "process":
// 2. A processor that does a (mock) geoip lookup, removes pii information
// (IP address), and repartitions by country.
var transformProcessor = new Processor <Null, string, string, string>
{
Name = "geo-lookup-processor",
BootstrapServers = brokerAddress,
InputTopic = simulatedWeblogTopic,
OutputTopic = filteredWeblogTopic,
ConsumeErrorTolerance = ErrorTolerance.All,
Function = (m) =>
{
try
{
var logline = m.Value;
var firstSpaceIndex = logline.IndexOf(' ');
if (firstSpaceIndex < 0)
{
throw new FormatException("unexpected logline format");
}
var ip = logline.Substring(0, firstSpaceIndex);
var country = MockGeoLookup.GetCountryFromIPAsync(ip);
var loglineWithoutIP = logline.Substring(firstSpaceIndex + 1);
var dateStart = loglineWithoutIP.IndexOf('[');
var dateEnd = loglineWithoutIP.IndexOf(']');
if (dateStart < 0 || dateEnd < 0 || dateEnd < dateStart)
{
throw new FormatException("unexpected logline format");
}
var requestInfo = loglineWithoutIP.Substring(dateEnd + 2);
return(new Message <string, string> {
Key = country, Value = requestInfo
});
}
catch (Exception)
{
// Unhandled exceptions in your processing function will cause the
// processor to terminate.
return(null); // null -> filter (don't write output message corresponding to input message).
}
}
};
transformProcessor.Start(instanceId, cts.Token);
break;
case "log":
// 3. A processor that just writes messages to stdout.
var consoleWriterProcessor = new Processor <string, string, Null, Null>
{
Name = "console-writer",
BootstrapServers = brokerAddress,
InputTopic = filteredWeblogTopic,
Function = (m) =>
{
Console.WriteLine($"{m.Key} ~~~ {m.Value}");
return(null);
}
};
consoleWriterProcessor.Start(instanceId, cts.Token);
break;
default:
Console.WriteLine($"unknown command {command}");
break;
}
}
