private static async Task <EventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger)
{
bool offsetInclusive = true;
EventHubClient client = EventHubClient.CreateFromConnectionString(partitionSettings.Hub.ConnectionString, partitionSettings.Hub.Path);
EventHubConsumerGroup consumerGroup = client.GetConsumerGroup(partitionSettings.Hub.ConsumerGroup);
if (partitionSettings.Hub.PrefetchCount.HasValue)
{
consumerGroup.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value;
}
// if we have a starting offset or if we're not configured to start reading from utc now, read from offset
if (!partitionSettings.Hub.StartFromNow || offset != EventHubConsumerGroup.StartOfStream)
{
logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
}
else
{
// to start reading from most recent data, we get the latest offset from the partition.
PartitionRuntimeInformation patitionInfo =
await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition);
offset = patitionInfo.LastEnqueuedOffset;
offsetInclusive = false;
logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
}
return(await consumerGroup.CreateReceiverAsync(partitionSettings.Partition, offset, offsetInclusive));
}
public EventHubAdapterReceiver(EventHubPartitionSettings settings,
Func <string, IStreamQueueCheckpointer <string>, Logger, IEventHubQueueCache> cacheFactory,
Func <string, Task <IStreamQueueCheckpointer <string> > > checkpointerFactory,
Logger baseLogger,
IEventHubReceiverMonitor monitor = null)
{
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (cacheFactory == null)
{
throw new ArgumentNullException(nameof(cacheFactory));
}
if (checkpointerFactory == null)
{
throw new ArgumentNullException(nameof(checkpointerFactory));
}
if (baseLogger == null)
{
throw new ArgumentNullException(nameof(baseLogger));
}
this.settings = settings;
this.cacheFactory = cacheFactory;
this.checkpointerFactory = checkpointerFactory;
this.baseLogger = baseLogger;
this.logger = baseLogger.GetSubLogger("receiver", "-");
this.monitor = monitor ?? new DefaultEventHubReceiverMonitor(settings.Hub.Path, settings.Partition, baseLogger.GetSubLogger("monitor", "-"));
}
public EventHubAdapterReceiver(EventHubPartitionSettings settings,
Func<string, IStreamQueueCheckpointer<string>, Logger, ITelemetryProducer, IEventHubQueueCache> cacheFactory,
Func<string, Task<IStreamQueueCheckpointer<string>>> checkpointerFactory,
Logger baseLogger,
IQueueAdapterReceiverMonitor monitor,
Factory<NodeConfiguration> getNodeConfig,
ITelemetryProducer telemetryProducer,
Func<EventHubPartitionSettings, string, Logger, ITelemetryProducer, Task<IEventHubReceiver>> eventHubReceiverFactory = null)
{
if (settings == null) throw new ArgumentNullException(nameof(settings));
if (cacheFactory == null) throw new ArgumentNullException(nameof(cacheFactory));
if (checkpointerFactory == null) throw new ArgumentNullException(nameof(checkpointerFactory));
if (baseLogger == null) throw new ArgumentNullException(nameof(baseLogger));
if (monitor == null) throw new ArgumentNullException(nameof(monitor));
if (telemetryProducer == null) throw new ArgumentNullException(nameof(telemetryProducer));
this.settings = settings;
this.cacheFactory = cacheFactory;
this.checkpointerFactory = checkpointerFactory;
this.baseLogger = baseLogger;
this.logger = baseLogger.GetSubLogger("receiver", "-");
this.monitor = monitor;
this.getNodeConfig = getNodeConfig;
this.telemetryProducer = telemetryProducer;
this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory;
}
private static async Task <IEventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger, ITelemetryProducer telemetryProducer)
{
bool offsetInclusive = true;
var connectionStringBuilder = new EventHubsConnectionStringBuilder(partitionSettings.Hub.ConnectionString)
{
EntityPath = partitionSettings.Hub.Path
};
EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
// if we have a starting offset or if we're not configured to start reading from utc now, read from offset
if (!partitionSettings.Hub.StartFromNow ||
offset != EventHubConstants.StartOfStream)
{
logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
}
else
{
// to start reading from most recent data, we get the latest offset from the partition.
EventHubPartitionRuntimeInformation partitionInfo =
await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition);
offset = partitionInfo.LastEnqueuedOffset;
offsetInclusive = false;
logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
}
PartitionReceiver receiver = client.CreateReceiver(partitionSettings.Hub.ConsumerGroup, partitionSettings.Partition, offset, offsetInclusive);
if (partitionSettings.Hub.PrefetchCount.HasValue)
{
receiver.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value;
}
return(new EventHubReceiverProxy(receiver));
}
public EventHubAdapterReceiver(EventHubPartitionSettings settings,
Func <string, IStreamQueueCheckpointer <string>, Logger, IEventHubQueueCache> cacheFactory,
Func <string, Task <IStreamQueueCheckpointer <string> > > checkpointerFactory,
Logger baseLogger,
IEventHubReceiverMonitor monitor,
Func <NodeConfiguration> getNodeConfig)
{
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (cacheFactory == null)
{
throw new ArgumentNullException(nameof(cacheFactory));
}
if (checkpointerFactory == null)
{
throw new ArgumentNullException(nameof(checkpointerFactory));
}
if (baseLogger == null)
{
throw new ArgumentNullException(nameof(baseLogger));
}
if (monitor == null)
{
throw new ArgumentNullException(nameof(monitor));
}
this.settings = settings;
this.cacheFactory = cacheFactory;
this.checkpointerFactory = checkpointerFactory;
this.baseLogger = baseLogger;
this.logger = baseLogger.GetSubLogger("receiver", "-");
this.monitor = monitor;
this.getNodeConfig = getNodeConfig;
}
private Task <IEventHubReceiver> EHGeneratorReceiverFactory(EventHubPartitionSettings settings, string offset, Logger logger)
{
var generator = new EventHubPartitionDataGenerator(logger, this.serviceProvider.GetRequiredService <SerializationManager>(), this.ehGeneratorSettings);
var generatorReceiver = new EventHubPartitionGeneratorReceiver(generator);
return(Task.FromResult <IEventHubReceiver>(generatorReceiver));
}
private EventHubAdapterReceiver MakeReceiver(QueueId queueId)
{
var config = new EventHubPartitionSettings
{
Hub = hubSettings,
Partition = streamQueueMapper.QueueToPartition(queueId),
};
Logger recieverLogger = logger.GetSubLogger($"{config.Partition}");
return(new EventHubAdapterReceiver(config, CacheFactory, CheckpointerFactory, recieverLogger, ReceiverMonitorFactory(config.Hub.Path, config.Partition, recieverLogger)));
}
private static async Task <EventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger)
#endif
{
bool offsetInclusive = true;
#if NETSTANDARD
var connectionStringBuilder = new EventHubsConnectionStringBuilder(partitionSettings.Hub.ConnectionString)
{
EntityPath = partitionSettings.Hub.Path
};
EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
#else
EventHubClient client = EventHubClient.CreateFromConnectionString(partitionSettings.Hub.ConnectionString, partitionSettings.Hub.Path);
EventHubConsumerGroup consumerGroup = client.GetConsumerGroup(partitionSettings.Hub.ConsumerGroup);
if (partitionSettings.Hub.PrefetchCount.HasValue)
{
consumerGroup.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value;
}
#endif
// if we have a starting offset or if we're not configured to start reading from utc now, read from offset
if (!partitionSettings.Hub.StartFromNow ||
offset != EventHubConstants.StartOfStream)
{
logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
}
else
{
// to start reading from most recent data, we get the latest offset from the partition.
#if NETSTANDARD
EventHubPartitionRuntimeInformation partitionInfo =
#else
PartitionRuntimeInformation partitionInfo =
#endif
await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition);
offset = partitionInfo.LastEnqueuedOffset;
offsetInclusive = false;
logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
}
#if NETSTANDARD
PartitionReceiver receiver = client.CreateReceiver(partitionSettings.Hub.ConsumerGroup, partitionSettings.Partition, offset, offsetInclusive);
if (partitionSettings.Hub.PrefetchCount.HasValue)
{
receiver.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value;
}
return(receiver);
#else
return(await consumerGroup.CreateReceiverAsync(partitionSettings.Partition, offset, offsetInclusive));
#endif
}
private EventHubAdapterReceiver MakeReceiver(QueueId queueId)
{
var config = new EventHubPartitionSettings
{
Hub = hubSettings,
Partition = streamQueueMapper.QueueToPartition(queueId),
};
Logger recieverLogger = logger.GetSubLogger($"{config.Partition}");
return(new EventHubAdapterReceiver(config, CacheFactory, CheckpointerFactory, recieverLogger, ReceiverMonitorFactory(config.Hub.Path, config.Partition, recieverLogger),
this.serviceProvider.GetRequiredService <Func <NodeConfiguration> >(),
this.EventHubReceiverFactory));
}
public EventHubReceiverProxy(EventHubPartitionSettings partitionSettings, string offset, ILogger logger)
{
var receiverOptions = new PartitionReceiverOptions();
if (partitionSettings.ReceiverOptions.PrefetchCount != null)
{
receiverOptions.PrefetchCount = partitionSettings.ReceiverOptions.PrefetchCount.Value;
}
receiverOptions.ConnectionOptions = new EventHubConnectionOptions {
TransportType = partitionSettings.Hub.EventHubsTransportType
};
var options = partitionSettings.Hub;
this.client = options.TokenCredential != null
? new PartitionReceiver(options.ConsumerGroup, partitionSettings.Partition, GetEventPosition(), options.FullyQualifiedNamespace, options.Path, options.TokenCredential, receiverOptions)
: new PartitionReceiver(options.ConsumerGroup, partitionSettings.Partition, GetEventPosition(), options.ConnectionString, options.Path, receiverOptions);
EventPosition GetEventPosition()
{
EventPosition eventPosition;
// If we have a starting offset, read from offset
if (offset != EventHubConstants.StartOfStream)
{
if (!long.TryParse(offset, out var longOffset))
{
throw new InvalidOperationException("Offset must be a number.");
}
logger.LogInformation("Starting to read from EventHub partition {0}-{1} at offset {2}", options.Path, partitionSettings.Partition, offset);
eventPosition = EventPosition.FromOffset(longOffset, true);
}
// else, if configured to start from now, start reading from most recent data
else if (partitionSettings.ReceiverOptions.StartFromNow)
{
eventPosition = EventPosition.Latest;
logger.LogInformation("Starting to read latest messages from EventHub partition {0}-{1}.", options.Path, partitionSettings.Partition);
}
else
// else, start reading from begining of the partition
{
eventPosition = EventPosition.Earliest;
logger.LogInformation("Starting to read messages from begining of EventHub partition {0}-{1}.", options.Path, partitionSettings.Partition);
}
return(eventPosition);
}
}
public EventHubAdapterReceiver(EventHubPartitionSettings settings,
Func <string, IStreamQueueCheckpointer <string>, ILoggerFactory, ITelemetryProducer, IEventHubQueueCache> cacheFactory,
Func <string, Task <IStreamQueueCheckpointer <string> > > checkpointerFactory,
ILoggerFactory loggerFactory,
IQueueAdapterReceiverMonitor monitor,
LoadSheddingOptions loadSheddingOptions,
ITelemetryProducer telemetryProducer,
Func <EventHubPartitionSettings, string, ILogger, ITelemetryProducer, IEventHubReceiver> eventHubReceiverFactory = null)
{
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (cacheFactory == null)
{
throw new ArgumentNullException(nameof(cacheFactory));
}
if (checkpointerFactory == null)
{
throw new ArgumentNullException(nameof(checkpointerFactory));
}
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
if (monitor == null)
{
throw new ArgumentNullException(nameof(monitor));
}
if (loadSheddingOptions == null)
{
throw new ArgumentNullException(nameof(loadSheddingOptions));
}
if (telemetryProducer == null)
{
throw new ArgumentNullException(nameof(telemetryProducer));
}
this.settings = settings;
this.cacheFactory = cacheFactory;
this.checkpointerFactory = checkpointerFactory;
this.loggerFactory = loggerFactory;
this.logger = this.loggerFactory.CreateLogger($"{this.GetType().FullName}.{settings.Hub.Path}.{settings.Partition}");
this.monitor = monitor;
this.telemetryProducer = telemetryProducer;
this.loadSheddingOptions = loadSheddingOptions;
this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory;
}
private EventHubAdapterReceiver MakeReceiver(QueueId queueId)
{
var config = new EventHubPartitionSettings
{
Hub = hubSettings,
Partition = this.streamQueueMapper.QueueToPartition(queueId),
};
var receiverMonitorDimensions = new EventHubReceiverMonitorDimensions
{
EventHubPartition = config.Partition,
EventHubPath = config.Hub.Path,
};
return(new EventHubAdapterReceiver(config, this.CacheFactory, this.CheckpointerFactory, this.loggerFactory, this.ReceiverMonitorFactory(receiverMonitorDimensions, this.loggerFactory, this.telemetryProducer),
this.serviceProvider.GetRequiredService <IOptions <LoadSheddingOptions> >().Value,
this.telemetryProducer,
this.EventHubReceiverFactory));
}
public EventHubAdapterReceiver(EventHubPartitionSettings partitionSettings,
Func <string, IStreamQueueCheckpointer <string>, Logger, IEventHubQueueCache> cacheFactory,
Func <string, Task <IStreamQueueCheckpointer <string> > > checkpointerFactory,
Logger logger)
{
this.cacheFactory = cacheFactory;
this.checkpointerFactory = checkpointerFactory;
baseLogger = logger;
this.logger = logger.GetSubLogger("-receiver");
settings = partitionSettings;
hubReceiveTimeMetric = $"Orleans.ServiceBus.EventHub.ReceiveTime_{settings.Hub.Path}";
partitionReceiveTimeMetric = $"Orleans.ServiceBus.EventHub.ReceiveTime_{settings.Hub.Path}-{settings.Partition}";
hubReadFailure = $"Orleans.ServiceBus.EventHub.ReadFailure_{settings.Hub.Path}";
partitionReadFailure = $"Orleans.ServiceBus.EventHub.ReadFailure_{settings.Hub.Path}-{settings.Partition}";
hubMessagesRecieved = $"Orleans.ServiceBus.EventHub.MessagesReceived_{settings.Hub.Path}";
partitionMessagesReceived = $"Orleans.ServiceBus.EventHub.MessagesReceived_{settings.Hub.Path}-{settings.Partition}";
hubAgeOfMessagesBeingProcessed = $"Orleans.ServiceBus.EventHub.AgeOfMessagesBeingProcessed_{settings.Hub.Path}";
partitionAgeOfMessagesBeingProcessed = $"Orleans.ServiceBus.EventHub.AgeOfMessagesBeingProcessed_{settings.Hub.Path}-{settings.Partition}";
}
private EventHubAdapterReceiver MakeReceiver(QueueId queueId)
{
var config = new EventHubPartitionSettings
{
Hub = hubSettings,
Partition = streamQueueMapper.QueueToPartition(queueId),
};
var receiverMonitorDimensions = new EventHubReceiverMonitorDimensions();
receiverMonitorDimensions.EventHubPartition = config.Partition;
receiverMonitorDimensions.EventHubPath = config.Hub.Path;
receiverMonitorDimensions.NodeConfig = this.serviceProvider.GetRequiredService <NodeConfiguration>();
receiverMonitorDimensions.GlobalConfig = this.serviceProvider.GetRequiredService <GlobalConfiguration>();
return(new EventHubAdapterReceiver(config, CacheFactory, CheckpointerFactory, loggerFactory, ReceiverMonitorFactory(receiverMonitorDimensions, loggerFactory, this.telemetryProducer),
this.serviceProvider.GetRequiredService <Factory <NodeConfiguration> >(),
this.telemetryProducer,
this.EventHubReceiverFactory));
}
private static IEventHubReceiver CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, ILogger logger, ITelemetryProducer telemetryProducer)
{
var connectionStringBuilder = new EventHubsConnectionStringBuilder(partitionSettings.Hub.ConnectionString)
{
EntityPath = partitionSettings.Hub.Path
};
EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
EventPosition eventPosition;
// If we have a starting offset, read from offset
if (offset != EventHubConstants.StartOfStream)
{
logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
eventPosition = EventPosition.FromOffset(offset, true);
}
// else, if configured to start from now, start reading from most recent data
else if (partitionSettings.ReceiverOptions.StartFromNow)
{
eventPosition = EventPosition.FromEnd();
logger.Info("Starting to read latest messages from EventHub partition {0}-{1}.", partitionSettings.Hub.Path, partitionSettings.Partition);
}
else
// else, start reading from begining of the partition
{
eventPosition = EventPosition.FromStart();
logger.Info("Starting to read messages from begining of EventHub partition {0}-{1}.", partitionSettings.Hub.Path, partitionSettings.Partition);
}
PartitionReceiver receiver = client.CreateReceiver(partitionSettings.Hub.ConsumerGroup, partitionSettings.Partition, eventPosition);
if (partitionSettings.ReceiverOptions.PrefetchCount.HasValue)
{
receiver.PrefetchCount = partitionSettings.ReceiverOptions.PrefetchCount.Value;
}
return(new EventHubReceiverProxy(receiver));
}
private static async Task <PartitionReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger)
private static IEventHubReceiver CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, ILogger logger, ITelemetryProducer telemetryProducer)
{
return(new EventHubReceiverProxy(partitionSettings, offset, logger));
}