async Task ConsumeForSomeTime(
ConcurrentQueue <string> receivedEvents,
Expression <Func <ConcurrentQueue <string>, bool> > completionExpression,
Expression <Func <ConcurrentQueue <string>, bool> > failExpression,
Func <string> errorDetailsFactory)
{
var consumer = Configure
.Consumer("default-group", c => c.UseKafka(KafkaTestConfig.Address))
.Logging(l => l.UseSerilog())
.Topics(t => t.Subscribe(_topic))
.Handle(async(messages, cancellationToken) =>
{
var strings = messages.Select(m => m.Body).OfType <string>().ToList();
Console.WriteLine($"Received these strings: {string.Join(", ", strings)}");
foreach (var str in strings)
{
receivedEvents.Enqueue(str);
}
})
.Positions(p =>
{
p.StoreInMemory(_positionsStorage);
var registrar = StandardConfigurer.Open(p);
registrar.Decorate(c => new WireTapPositionsManager(c.Get <IPositionManager>()));
})
.Start();
using (consumer)
{
try
{
await receivedEvents.WaitOrDie(completionExpression, failExpression, timeoutSeconds : 10);
}
catch (TimeoutException exception)
{
throw new TimeoutException($"Failed with details: {errorDetailsFactory()}", exception);
}
catch (Exception exception)
{
throw new ApplicationException($"Failed with details: {errorDetailsFactory()}", exception);
}
await Task.Delay(TimeSpan.FromSeconds(1));
var compiledExpression = completionExpression.Compile();
Assert.That(compiledExpression(receivedEvents), Is.True, $@"The expression
{compiledExpression}
was no longer true!
Failed with details: {errorDetailsFactory()}
");
}
}
/// <summary>
/// Configures Topos to use Microsoft's FASTER Log and the file system as the event store
/// </summary>
public static FasterProducerConfigurationBuilder UseFileSystem(this StandardConfigurer <IProducerImplementation> configurer, string directoryPath)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
var builder = new FasterProducerConfigurationBuilder();
CheckDirectoryPath(directoryPath);
StandardConfigurer.Open(configurer)
.Register(c => new FasterLogProducerImplementation(
loggerFactory: c.Get <ILoggerFactory>(),
deviceManager: c.Get <IDeviceManager>(),
logEntrySerializer: c.Get <ILogEntrySerializer>(),
eventExpirationHelper: c.Get <EventExpirationHelper>()
))
.Other <IDeviceManager>().Register(c => new DefaultDeviceManager(
loggerFactory: c.Get <ILoggerFactory>(),
directoryPath: directoryPath
))
.Other <ILogEntrySerializer>().Register(_ => new ProtobufLogEntrySerializer())
.Other <EventExpirationHelper>().Register(c => new EventExpirationHelper(
loggerFactory: c.Get <ILoggerFactory>(),
deviceManager: c.Get <IDeviceManager>(),
maxAgesPerTopic: builder.GetMaxAges(),
logEntrySerializer: c.Get <ILogEntrySerializer>()
));
return(builder);
}
public static KafkaConsumerConfigurationBuilder UseKafka(this StandardConfigurer <IConsumerImplementation> configurer, IEnumerable <string> bootstrapServers)
{
var builder = new KafkaConsumerConfigurationBuilder();
StandardConfigurer.Open(configurer)
.Register(c =>
{
var loggerFactory = c.Get <ILoggerFactory>();
var topics = c.Has <Topics>() ? c.Get <Topics>() : new Topics();
var group = c.Get <GroupId>();
var consumerDispatcher = c.Get <IConsumerDispatcher>();
var positionManager = c.Get <IPositionManager>(errorMessage: @"The Kafka consumer needs access to a positions manager, so it can figure out which offsets to pick up from when starting up.");
var consumerContext = c.Get <ConsumerContext>();
var partitionsAssignedHandler = builder.GetPartitionsAssignedHandler();
var partitionsRevokedHandler = builder.GetPartitionsRevokedHandler();
var startPosition = c.Has <PositionsStorageConfigurationExtensions.ExplicitlySetInitialPosition>() ? c.Get <PositionsStorageConfigurationExtensions.ExplicitlySetInitialPosition>().Position : StartFromPosition.Beginning;
return(new KafkaConsumerImplementation(
loggerFactory: loggerFactory,
address: string.Join("; ", bootstrapServers),
topics: topics,
group: group.Id,
consumerDispatcher: consumerDispatcher,
positionManager: positionManager,
context: consumerContext,
configurationCustomizer: config => builder.Apply(config),
partitionsAssignedHandler: partitionsAssignedHandler,
partitionsRevokedHandler: partitionsRevokedHandler,
startPosition: startPosition
));
});
return(builder);
}
public static void UseSimple(this StandardConfigurer <ITopicMapper> configurer)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
StandardConfigurer.Open(configurer).Register(c => new SimpleTopicMapper());
}
public static void UseSerilog(this StandardConfigurer <ILoggerFactory> configurer)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
StandardConfigurer.Open(configurer).Register(_ => new SerilogLoggerFactory());
}
/// <summary>
/// Configures Topos to use .NET's built-in <see cref="JsonSerializer"/> to serialize/deserialize messages
/// </summary>
public static void UseSystemTextJson(this StandardConfigurer <IMessageSerializer> configurer)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
StandardConfigurer.Open(configurer).Register(_ => new SystemTextJsonSerializer());
}
public static KafkaProducerConfigurationBuilder UseKafka(this StandardConfigurer <IProducerImplementation> configurer, IEnumerable <string> bootstrapServers)
{
var builder = new KafkaProducerConfigurationBuilder();
StandardConfigurer.Open(configurer)
.Register(c => RegisterProducerImplementation(bootstrapServers, c, builder));
return(builder);
}
public static void UseNewtonsoftJson(this StandardConfigurer <IMessageSerializer> configurer, JsonSerializerSettings settings = null, Encoding encoding = null)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
StandardConfigurer.Open(configurer).Register(_ => new JsonSerializer(settings: settings, encoding: encoding));
}
public static void DatafordelerEventDeserializer(this StandardConfigurer <IMessageSerializer> configurer)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
StandardConfigurer.Open(configurer)
.Register(c => new DatafordelerEventDeserializer());
}
public static void RouteNetwork(this StandardConfigurer <IMessageSerializer> configurer)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
StandardConfigurer.Open(configurer)
.Register(c => new RouteNetworkSerializer(new InfoMapper()));
}
public static void GenericEventDeserializer <BaseEventType>(this StandardConfigurer <IMessageSerializer> configurer)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
StandardConfigurer.Open(configurer)
.Register(c => new GenericEventDeserializer <BaseEventType>());
}
static void ChangeCompactionIntervalTo(StandardConfigurer <IProducerImplementation> p, TimeSpan interval)
{
StandardConfigurer.Open(p)
.Other <EventExpirationHelper>()
.Decorate(c =>
{
var helper = c.Get <EventExpirationHelper>();
helper.CompactionInterval = interval;
return(helper);
});
}
public static void SetInitialPosition(this StandardConfigurer <IPositionManager> configurer, StartFromPosition startFromPosition)
{
var registrar = StandardConfigurer.Open(configurer);
if (registrar.Other <ExplicitlySetInitialPosition>().HasService())
{
throw new InvalidOperationException($"Cannot set resume position to {startFromPosition}, because it has already been set!");
}
registrar.Other <ExplicitlySetInitialPosition>().Register(c => new ExplicitlySetInitialPosition(startFromPosition));
registrar.Decorate(c => new InitialPositionDecorator(c.Get <IPositionManager>(), startFromPosition));
}
public static void UseInMemory(this StandardConfigurer <IConsumerImplementation> configurer, InMemEventBroker eventBroker)
{
StandardConfigurer.Open(configurer)
.Register(c =>
{
var loggerFactory = c.Get <ILoggerFactory>();
var topics = c.Has <Topics>() ? c.Get <Topics>() : new Topics();
var consumerDispatcher = c.Get <IConsumerDispatcher>();
var consumerContext = c.Get <ConsumerContext>();
return(new InMemConsumerImplementation(eventBroker, loggerFactory, topics, consumerDispatcher, consumerContext));
});
}
public static void StoreInFileSystem(this StandardConfigurer <IPositionManager> configurer, string directoryPath)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
if (directoryPath == null)
{
throw new ArgumentNullException(nameof(directoryPath));
}
var registrar = StandardConfigurer.Open(configurer);
registrar.Register(_ => new FileSystemPositionsManager(directoryPath));
}
/// <summary>
/// Configures Topos to stores its consumer positions in blobs. Individual blobs will be created for each relevant topic/partition.
/// </summary>
public static void StoreInAzureBlobs(this StandardConfigurer <IPositionManager> configurer, CloudStorageAccount storageAccount, string containerName)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
if (storageAccount == null)
{
throw new ArgumentNullException(nameof(storageAccount));
}
if (containerName == null)
{
throw new ArgumentNullException(nameof(containerName));
}
var registrar = StandardConfigurer.Open(configurer);
registrar.Register(_ => new AzureBlobsPositionManager(storageAccount, containerName));
}
/// <summary>
/// Configures Topos to stores its consumer positions in documents in MongoDB. Individual documents will be created for each relevant topic with fields for each partition.
/// </summary>
public static void StoreInMongoDb(this StandardConfigurer <IPositionManager> configurer, IMongoDatabase database, string collectionName)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
if (database == null)
{
throw new ArgumentNullException(nameof(database));
}
if (collectionName == null)
{
throw new ArgumentNullException(nameof(collectionName));
}
var registrar = StandardConfigurer.Open(configurer);
registrar.Register(_ => new MongoDbPositionManager(database, collectionName));
}
public static void StoreInPostgreSql(
this StandardConfigurer <IPositionManager> configurer,
string connectionString,
string consumerGroup)
{
if (configurer is null)
{
throw new ArgumentNullException(nameof(configurer));
}
if (string.IsNullOrEmpty(connectionString))
{
throw new ArgumentException($"{nameof(connectionString)} cannot be null or empty.");
}
if (string.IsNullOrEmpty(consumerGroup))
{
throw new ArgumentException($"{nameof(consumerGroup)} cannot be null or empty.");
}
StandardConfigurer.Open(configurer)
.Register(_ => new PostgreSqlPositionManager(connectionString, consumerGroup));
}
/// <summary>
/// Configures Topos to use Microsoft's FASTER Log and the file system as the event store
/// </summary>
public static void UseFileSystem(this StandardConfigurer <IConsumerImplementation> configurer, string directoryPath)
{
if (configurer == null)
{
throw new ArgumentNullException(nameof(configurer));
}
CheckDirectoryPath(directoryPath);
StandardConfigurer.Open(configurer)
.Register(c => new FasterLogConsumerImplementation(
loggerFactory: c.Get <ILoggerFactory>(),
deviceManager: c.Get <IDeviceManager>(),
logEntrySerializer: c.Get <ILogEntrySerializer>(),
topics: c.Has <Topics>() ? c.Get <Topics>() : new Topics(),
consumerDispatcher: c.Get <IConsumerDispatcher>(),
positionManager: c.Get <IPositionManager>()
))
.Other <IDeviceManager>().Register(c => new DefaultDeviceManager(
loggerFactory: c.Get <ILoggerFactory>(),
directoryPath: directoryPath
))
.Other <ILogEntrySerializer>().Register(_ => new ProtobufLogEntrySerializer());
}
public static void UseConsole(this StandardConfigurer <ILoggerFactory> configurer, LogLevel minimumLogLevel = LogLevel.Debug)
{
StandardConfigurer.Open(configurer).Register(c => new ConsoleLoggerFactory(minimumLogLevel));
}
public static void Use(this StandardConfigurer <ILoggerFactory> configurer, ILoggerFactory loggerFactory)
{
StandardConfigurer.Open(configurer).Decorate(c => loggerFactory);
}
public static void UseInMemory(this StandardConfigurer <IProducerImplementation> configurer, InMemEventBroker eventBroker)
{
StandardConfigurer.Open(configurer)
.Register(_ => new InMemProducerImplementation(eventBroker));
}
public static void StoreInMemory(this StandardConfigurer <IPositionManager> configurer, InMemPositionsStorage positionsStorage = null)
{
var registrar = StandardConfigurer.Open(configurer);
registrar.Register(_ => new InMemPositionsManager(positionsStorage ?? new InMemPositionsStorage()));
}