private static async Task <EventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger) { bool offsetInclusive = true; EventHubClient client = EventHubClient.CreateFromConnectionString(partitionSettings.Hub.ConnectionString, partitionSettings.Hub.Path); EventHubConsumerGroup consumerGroup = client.GetConsumerGroup(partitionSettings.Hub.ConsumerGroup); if (partitionSettings.Hub.PrefetchCount.HasValue) { consumerGroup.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value; } // if we have a starting offset or if we're not configured to start reading from utc now, read from offset if (!partitionSettings.Hub.StartFromNow || offset != EventHubConsumerGroup.StartOfStream) { logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset); } else { // to start reading from most recent data, we get the latest offset from the partition. PartitionRuntimeInformation patitionInfo = await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition); offset = patitionInfo.LastEnqueuedOffset; offsetInclusive = false; logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset); } return(await consumerGroup.CreateReceiverAsync(partitionSettings.Partition, offset, offsetInclusive)); }
public EventHubAdapterReceiver(EventHubPartitionSettings settings, Func <string, IStreamQueueCheckpointer <string>, Logger, IEventHubQueueCache> cacheFactory, Func <string, Task <IStreamQueueCheckpointer <string> > > checkpointerFactory, Logger baseLogger, IEventHubReceiverMonitor monitor = null) { if (settings == null) { throw new ArgumentNullException(nameof(settings)); } if (cacheFactory == null) { throw new ArgumentNullException(nameof(cacheFactory)); } if (checkpointerFactory == null) { throw new ArgumentNullException(nameof(checkpointerFactory)); } if (baseLogger == null) { throw new ArgumentNullException(nameof(baseLogger)); } this.settings = settings; this.cacheFactory = cacheFactory; this.checkpointerFactory = checkpointerFactory; this.baseLogger = baseLogger; this.logger = baseLogger.GetSubLogger("receiver", "-"); this.monitor = monitor ?? new DefaultEventHubReceiverMonitor(settings.Hub.Path, settings.Partition, baseLogger.GetSubLogger("monitor", "-")); }
public EventHubAdapterReceiver(EventHubPartitionSettings settings, Func<string, IStreamQueueCheckpointer<string>, Logger, ITelemetryProducer, IEventHubQueueCache> cacheFactory, Func<string, Task<IStreamQueueCheckpointer<string>>> checkpointerFactory, Logger baseLogger, IQueueAdapterReceiverMonitor monitor, Factory<NodeConfiguration> getNodeConfig, ITelemetryProducer telemetryProducer, Func<EventHubPartitionSettings, string, Logger, ITelemetryProducer, Task<IEventHubReceiver>> eventHubReceiverFactory = null) { if (settings == null) throw new ArgumentNullException(nameof(settings)); if (cacheFactory == null) throw new ArgumentNullException(nameof(cacheFactory)); if (checkpointerFactory == null) throw new ArgumentNullException(nameof(checkpointerFactory)); if (baseLogger == null) throw new ArgumentNullException(nameof(baseLogger)); if (monitor == null) throw new ArgumentNullException(nameof(monitor)); if (telemetryProducer == null) throw new ArgumentNullException(nameof(telemetryProducer)); this.settings = settings; this.cacheFactory = cacheFactory; this.checkpointerFactory = checkpointerFactory; this.baseLogger = baseLogger; this.logger = baseLogger.GetSubLogger("receiver", "-"); this.monitor = monitor; this.getNodeConfig = getNodeConfig; this.telemetryProducer = telemetryProducer; this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory; }
private static async Task <IEventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger, ITelemetryProducer telemetryProducer) { bool offsetInclusive = true; var connectionStringBuilder = new EventHubsConnectionStringBuilder(partitionSettings.Hub.ConnectionString) { EntityPath = partitionSettings.Hub.Path }; EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString()); // if we have a starting offset or if we're not configured to start reading from utc now, read from offset if (!partitionSettings.Hub.StartFromNow || offset != EventHubConstants.StartOfStream) { logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset); } else { // to start reading from most recent data, we get the latest offset from the partition. EventHubPartitionRuntimeInformation partitionInfo = await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition); offset = partitionInfo.LastEnqueuedOffset; offsetInclusive = false; logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset); } PartitionReceiver receiver = client.CreateReceiver(partitionSettings.Hub.ConsumerGroup, partitionSettings.Partition, offset, offsetInclusive); if (partitionSettings.Hub.PrefetchCount.HasValue) { receiver.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value; } return(new EventHubReceiverProxy(receiver)); }
public EventHubAdapterReceiver(EventHubPartitionSettings settings, Func <string, IStreamQueueCheckpointer <string>, Logger, IEventHubQueueCache> cacheFactory, Func <string, Task <IStreamQueueCheckpointer <string> > > checkpointerFactory, Logger baseLogger, IEventHubReceiverMonitor monitor, Func <NodeConfiguration> getNodeConfig) { if (settings == null) { throw new ArgumentNullException(nameof(settings)); } if (cacheFactory == null) { throw new ArgumentNullException(nameof(cacheFactory)); } if (checkpointerFactory == null) { throw new ArgumentNullException(nameof(checkpointerFactory)); } if (baseLogger == null) { throw new ArgumentNullException(nameof(baseLogger)); } if (monitor == null) { throw new ArgumentNullException(nameof(monitor)); } this.settings = settings; this.cacheFactory = cacheFactory; this.checkpointerFactory = checkpointerFactory; this.baseLogger = baseLogger; this.logger = baseLogger.GetSubLogger("receiver", "-"); this.monitor = monitor; this.getNodeConfig = getNodeConfig; }
private Task <IEventHubReceiver> EHGeneratorReceiverFactory(EventHubPartitionSettings settings, string offset, Logger logger) { var generator = new EventHubPartitionDataGenerator(logger, this.serviceProvider.GetRequiredService <SerializationManager>(), this.ehGeneratorSettings); var generatorReceiver = new EventHubPartitionGeneratorReceiver(generator); return(Task.FromResult <IEventHubReceiver>(generatorReceiver)); }
private EventHubAdapterReceiver MakeReceiver(QueueId queueId) { var config = new EventHubPartitionSettings { Hub = hubSettings, Partition = streamQueueMapper.QueueToPartition(queueId), }; Logger recieverLogger = logger.GetSubLogger($"{config.Partition}"); return(new EventHubAdapterReceiver(config, CacheFactory, CheckpointerFactory, recieverLogger, ReceiverMonitorFactory(config.Hub.Path, config.Partition, recieverLogger))); }
private static async Task <EventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger) #endif { bool offsetInclusive = true; #if NETSTANDARD var connectionStringBuilder = new EventHubsConnectionStringBuilder(partitionSettings.Hub.ConnectionString) { EntityPath = partitionSettings.Hub.Path }; EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString()); #else EventHubClient client = EventHubClient.CreateFromConnectionString(partitionSettings.Hub.ConnectionString, partitionSettings.Hub.Path); EventHubConsumerGroup consumerGroup = client.GetConsumerGroup(partitionSettings.Hub.ConsumerGroup); if (partitionSettings.Hub.PrefetchCount.HasValue) { consumerGroup.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value; } #endif // if we have a starting offset or if we're not configured to start reading from utc now, read from offset if (!partitionSettings.Hub.StartFromNow || offset != EventHubConstants.StartOfStream) { logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset); } else { // to start reading from most recent data, we get the latest offset from the partition. #if NETSTANDARD EventHubPartitionRuntimeInformation partitionInfo = #else PartitionRuntimeInformation partitionInfo = #endif await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition); offset = partitionInfo.LastEnqueuedOffset; offsetInclusive = false; logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset); } #if NETSTANDARD PartitionReceiver receiver = client.CreateReceiver(partitionSettings.Hub.ConsumerGroup, partitionSettings.Partition, offset, offsetInclusive); if (partitionSettings.Hub.PrefetchCount.HasValue) { receiver.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value; } return(receiver); #else return(await consumerGroup.CreateReceiverAsync(partitionSettings.Partition, offset, offsetInclusive)); #endif }
private EventHubAdapterReceiver MakeReceiver(QueueId queueId) { var config = new EventHubPartitionSettings { Hub = hubSettings, Partition = streamQueueMapper.QueueToPartition(queueId), }; Logger recieverLogger = logger.GetSubLogger($"{config.Partition}"); return(new EventHubAdapterReceiver(config, CacheFactory, CheckpointerFactory, recieverLogger, ReceiverMonitorFactory(config.Hub.Path, config.Partition, recieverLogger), this.serviceProvider.GetRequiredService <Func <NodeConfiguration> >(), this.EventHubReceiverFactory)); }
public EventHubReceiverProxy(EventHubPartitionSettings partitionSettings, string offset, ILogger logger) { var receiverOptions = new PartitionReceiverOptions(); if (partitionSettings.ReceiverOptions.PrefetchCount != null) { receiverOptions.PrefetchCount = partitionSettings.ReceiverOptions.PrefetchCount.Value; } receiverOptions.ConnectionOptions = new EventHubConnectionOptions { TransportType = partitionSettings.Hub.EventHubsTransportType }; var options = partitionSettings.Hub; this.client = options.TokenCredential != null ? new PartitionReceiver(options.ConsumerGroup, partitionSettings.Partition, GetEventPosition(), options.FullyQualifiedNamespace, options.Path, options.TokenCredential, receiverOptions) : new PartitionReceiver(options.ConsumerGroup, partitionSettings.Partition, GetEventPosition(), options.ConnectionString, options.Path, receiverOptions); EventPosition GetEventPosition() { EventPosition eventPosition; // If we have a starting offset, read from offset if (offset != EventHubConstants.StartOfStream) { if (!long.TryParse(offset, out var longOffset)) { throw new InvalidOperationException("Offset must be a number."); } logger.LogInformation("Starting to read from EventHub partition {0}-{1} at offset {2}", options.Path, partitionSettings.Partition, offset); eventPosition = EventPosition.FromOffset(longOffset, true); } // else, if configured to start from now, start reading from most recent data else if (partitionSettings.ReceiverOptions.StartFromNow) { eventPosition = EventPosition.Latest; logger.LogInformation("Starting to read latest messages from EventHub partition {0}-{1}.", options.Path, partitionSettings.Partition); } else // else, start reading from begining of the partition { eventPosition = EventPosition.Earliest; logger.LogInformation("Starting to read messages from begining of EventHub partition {0}-{1}.", options.Path, partitionSettings.Partition); } return(eventPosition); } }
public EventHubAdapterReceiver(EventHubPartitionSettings settings, Func <string, IStreamQueueCheckpointer <string>, ILoggerFactory, ITelemetryProducer, IEventHubQueueCache> cacheFactory, Func <string, Task <IStreamQueueCheckpointer <string> > > checkpointerFactory, ILoggerFactory loggerFactory, IQueueAdapterReceiverMonitor monitor, LoadSheddingOptions loadSheddingOptions, ITelemetryProducer telemetryProducer, Func <EventHubPartitionSettings, string, ILogger, ITelemetryProducer, IEventHubReceiver> eventHubReceiverFactory = null) { if (settings == null) { throw new ArgumentNullException(nameof(settings)); } if (cacheFactory == null) { throw new ArgumentNullException(nameof(cacheFactory)); } if (checkpointerFactory == null) { throw new ArgumentNullException(nameof(checkpointerFactory)); } if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } if (monitor == null) { throw new ArgumentNullException(nameof(monitor)); } if (loadSheddingOptions == null) { throw new ArgumentNullException(nameof(loadSheddingOptions)); } if (telemetryProducer == null) { throw new ArgumentNullException(nameof(telemetryProducer)); } this.settings = settings; this.cacheFactory = cacheFactory; this.checkpointerFactory = checkpointerFactory; this.loggerFactory = loggerFactory; this.logger = this.loggerFactory.CreateLogger($"{this.GetType().FullName}.{settings.Hub.Path}.{settings.Partition}"); this.monitor = monitor; this.telemetryProducer = telemetryProducer; this.loadSheddingOptions = loadSheddingOptions; this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory; }
private EventHubAdapterReceiver MakeReceiver(QueueId queueId) { var config = new EventHubPartitionSettings { Hub = hubSettings, Partition = this.streamQueueMapper.QueueToPartition(queueId), }; var receiverMonitorDimensions = new EventHubReceiverMonitorDimensions { EventHubPartition = config.Partition, EventHubPath = config.Hub.Path, }; return(new EventHubAdapterReceiver(config, this.CacheFactory, this.CheckpointerFactory, this.loggerFactory, this.ReceiverMonitorFactory(receiverMonitorDimensions, this.loggerFactory, this.telemetryProducer), this.serviceProvider.GetRequiredService <IOptions <LoadSheddingOptions> >().Value, this.telemetryProducer, this.EventHubReceiverFactory)); }
public EventHubAdapterReceiver(EventHubPartitionSettings partitionSettings, Func <string, IStreamQueueCheckpointer <string>, Logger, IEventHubQueueCache> cacheFactory, Func <string, Task <IStreamQueueCheckpointer <string> > > checkpointerFactory, Logger logger) { this.cacheFactory = cacheFactory; this.checkpointerFactory = checkpointerFactory; baseLogger = logger; this.logger = logger.GetSubLogger("-receiver"); settings = partitionSettings; hubReceiveTimeMetric = $"Orleans.ServiceBus.EventHub.ReceiveTime_{settings.Hub.Path}"; partitionReceiveTimeMetric = $"Orleans.ServiceBus.EventHub.ReceiveTime_{settings.Hub.Path}-{settings.Partition}"; hubReadFailure = $"Orleans.ServiceBus.EventHub.ReadFailure_{settings.Hub.Path}"; partitionReadFailure = $"Orleans.ServiceBus.EventHub.ReadFailure_{settings.Hub.Path}-{settings.Partition}"; hubMessagesRecieved = $"Orleans.ServiceBus.EventHub.MessagesReceived_{settings.Hub.Path}"; partitionMessagesReceived = $"Orleans.ServiceBus.EventHub.MessagesReceived_{settings.Hub.Path}-{settings.Partition}"; hubAgeOfMessagesBeingProcessed = $"Orleans.ServiceBus.EventHub.AgeOfMessagesBeingProcessed_{settings.Hub.Path}"; partitionAgeOfMessagesBeingProcessed = $"Orleans.ServiceBus.EventHub.AgeOfMessagesBeingProcessed_{settings.Hub.Path}-{settings.Partition}"; }
private EventHubAdapterReceiver MakeReceiver(QueueId queueId) { var config = new EventHubPartitionSettings { Hub = hubSettings, Partition = streamQueueMapper.QueueToPartition(queueId), }; var receiverMonitorDimensions = new EventHubReceiverMonitorDimensions(); receiverMonitorDimensions.EventHubPartition = config.Partition; receiverMonitorDimensions.EventHubPath = config.Hub.Path; receiverMonitorDimensions.NodeConfig = this.serviceProvider.GetRequiredService <NodeConfiguration>(); receiverMonitorDimensions.GlobalConfig = this.serviceProvider.GetRequiredService <GlobalConfiguration>(); return(new EventHubAdapterReceiver(config, CacheFactory, CheckpointerFactory, loggerFactory, ReceiverMonitorFactory(receiverMonitorDimensions, loggerFactory, this.telemetryProducer), this.serviceProvider.GetRequiredService <Factory <NodeConfiguration> >(), this.telemetryProducer, this.EventHubReceiverFactory)); }
private static IEventHubReceiver CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, ILogger logger, ITelemetryProducer telemetryProducer) { var connectionStringBuilder = new EventHubsConnectionStringBuilder(partitionSettings.Hub.ConnectionString) { EntityPath = partitionSettings.Hub.Path }; EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString()); EventPosition eventPosition; // If we have a starting offset, read from offset if (offset != EventHubConstants.StartOfStream) { logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset); eventPosition = EventPosition.FromOffset(offset, true); } // else, if configured to start from now, start reading from most recent data else if (partitionSettings.ReceiverOptions.StartFromNow) { eventPosition = EventPosition.FromEnd(); logger.Info("Starting to read latest messages from EventHub partition {0}-{1}.", partitionSettings.Hub.Path, partitionSettings.Partition); } else // else, start reading from begining of the partition { eventPosition = EventPosition.FromStart(); logger.Info("Starting to read messages from begining of EventHub partition {0}-{1}.", partitionSettings.Hub.Path, partitionSettings.Partition); } PartitionReceiver receiver = client.CreateReceiver(partitionSettings.Hub.ConsumerGroup, partitionSettings.Partition, eventPosition); if (partitionSettings.ReceiverOptions.PrefetchCount.HasValue) { receiver.PrefetchCount = partitionSettings.ReceiverOptions.PrefetchCount.Value; } return(new EventHubReceiverProxy(receiver)); }
private static async Task <PartitionReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger)
private static IEventHubReceiver CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, ILogger logger, ITelemetryProducer telemetryProducer) { return(new EventHubReceiverProxy(partitionSettings, offset, logger)); }