public void DetermineOwnership(int id, int quantity) { var logger = GetLogger("testing"); var manager = new FixedPartitionManager(logger, Environment.GetEnvironmentVariable("iotc_eventhub"), id); manager.InitializeAsync(quantity).GetAwaiter().GetResult(); var partitions = manager.GetPartitions(); var ownership = new Dictionary <string, bool>(); var ownedPartitions = manager.GetOwnedPartitions(); for (var index = 0; index < partitions.Length; index++) { var partitionId = partitions[index]; var isOwner = manager.IsOwner(partitionId); ownership.Add(partitionId, isOwner); } for (var index = 0; index < partitions.Length; index++) { var partitionId = partitions[index]; var shouldOwn = (index % quantity == id); Assert.IsTrue(ownership[partitionId] == shouldOwn); } }
public void SimpleManagerTest() { var connectionString = Environment.GetEnvironmentVariable("ConnectionString"); // Retrieve the partition information from the Event or IoT Hub var client = EventHubClient.CreateFromConnectionString(connectionString); client.RetryPolicy = new RetryExponential(TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(10), 10); var runtimeInformation = client.GetRuntimeInformationAsync().GetAwaiter().GetResult(); var partitions = runtimeInformation.PartitionIds.OrderBy(item => item).ToArray(); var logger = new LoggerFactory().CreateLogger("test"); var manager = new FixedPartitionManager(logger, connectionString, 0); manager.InitializeAsync(2).GetAwaiter().GetResult(); var managerPartitions = manager.GetPartitions(); var managerOwnedPartitions = manager.GetPartitions(); var isOwned = manager.IsOwner("0"); Assert.IsTrue(managerPartitions.Length == partitions.Length, "Partition counts should match"); Assert.IsTrue(managerPartitions.Intersect(partitions).ToArray().Length == partitions.Length, "Intersecting partition should include all partitions"); var partitionHalf = (int)(partitions.Length / 2) - 1; Assert.IsTrue(Math.Abs(managerOwnedPartitions.Length - partitionHalf) <= 1, "Owned partitions should be approximately half in case of odd numbers"); Assert.IsTrue(Math.Abs(managerPartitions.Intersect(managerOwnedPartitions).ToArray().Length - partitionHalf) <= 1, "Intersecting owned partition should include all partitions"); Assert.IsTrue(isOwned, "Ownership of partition 0 is expected based on the known algorithm. If the algorithm of modulo fails this needs adjusted"); }
/// <summary> /// Initializes a new instance of the type /// </summary> /// <param name="logger">The logger to write debugging and diagnostics information to</param> /// <param name="metricFactory">A factory to create metric containers from</param> /// <param name="connectionString">The connection string to the Event Hub</param> /// <param name="managerId">The zero based index of the fixed partition manager</param> /// <param name="managerCount">The number of fixed partition managers that cooperating</param> public FixedLeaseManager(ILogger logger, IMetricFactory metricFactory, string connectionString, int managerId, int managerCount) { _partitionManager = new FixedPartitionManager(logger, connectionString, managerId); _managerCount = managerCount; _updateManagerCounter = metricFactory.CreateCounter("flm-update-counter", "The number of times the manager count has been updated", false, new string[0]); _ownershipClaimCounter = metricFactory.CreateCounter("flm-ownership-claim-counter", "The number of times a partition claim request has been executed (multiples per request)", false, new string[0]); _listOwnershipCounter = metricFactory.CreateCounter("flm-ownership-list-counter", "The number of times a partition claim list request has been executed", false, new string[0]); }
/// <inheritdoc /> protected sealed override async Task ExecuteAsync(CancellationToken cancellationToken) { Logger.LogInformation("Started execution loop"); // Once the initial tasks are completed indicate the service is now available IsContainerAvailable = true; var checkpointStoreRoot = Environment.GetEnvironmentVariable("CheckpointStore"); var eventHubConnectionString = await GetEventHubConnectionStringAsync().ConfigureAwait(false); var checkpointConnectionString = await GetStorageConnectionStringAsync().ConfigureAwait(false); var consumerGroupName = Environment.GetEnvironmentVariable("ConsumerGroup"); var checkpointContainerName = Environment.GetEnvironmentVariable("CheckpointContainerName"); var epochContainerName = Environment.GetEnvironmentVariable("EpochContainerName"); var eventProcessorHostName = Environment.GetEnvironmentVariable("EventProcessorHostName"); if (!int.TryParse(Environment.GetEnvironmentVariable("LeaseDurationSeconds"), out int leaseDurationSeconds) || leaseDurationSeconds < 20) { leaseDurationSeconds = 20; } if (!int.TryParse(Environment.GetEnvironmentVariable("LeaseRenewalSeconds"), out int leaseRenewalSeconds) || leaseRenewalSeconds < 10 || leaseRenewalSeconds >= leaseDurationSeconds) { leaseRenewalSeconds = 10; } if (!int.TryParse(Environment.GetEnvironmentVariable("ReceiveTimeoutSeconds"), out int receiveTimeout) || receiveTimeout < 5) { receiveTimeout = 60; } if (!int.TryParse(Environment.GetEnvironmentVariable("PrefetchCount"), out int prefetchCount) || prefetchCount < 0) { prefetchCount = 0; } if (!int.TryParse(Environment.GetEnvironmentVariable("MaxBatchSize"), out int maxBatchSize) || maxBatchSize < 1) { maxBatchSize = 1; } if (!int.TryParse(Environment.GetEnvironmentVariable("IterationSeconds"), out int iterationSeconds) || iterationSeconds < 1) { iterationSeconds = 1; } var iterationDelay = TimeSpan.FromSeconds(iterationSeconds); Logger.LogInformation("Creating fixed partition manager"); _partitionManager = new FixedPartitionManager(_loggerFactory.CreateLogger(PartitionManagerLoggerName), eventHubConnectionString, KubernetesStatefulSetIndex.Value); Logger.LogInformation("Initializing fixed partition manager"); await _partitionManager.InitializeAsync(KubernetesDesiredReplicaCount.Value).ConfigureAwait(true); Logger.LogInformation("Creating epoch reader"); var epochRecorder = new AzureStorageEpochRecorder(_loggerFactory.CreateLogger(EpochRecorderLoggerName), MetricFactory, consumerGroupName, checkpointConnectionString, epochContainerName, null); Logger.LogInformation("Creating fixed lease manager"); var leaseManager = new FixedLeaseManager(_loggerFactory.CreateLogger(LeaseManagerLoggerName), _partitionManager, epochRecorder); Logger.LogInformation("Creating Azure storage checkpoint manager"); var checkpointManager = new AzureStorageCheckpointManager(_loggerFactory.CreateLogger(CheckpointManagerLoggerName), MetricFactory, checkpointConnectionString, checkpointContainerName, null); Logger.LogInformation("Building connection string"); var builder = new EventHubsConnectionStringBuilder(eventHubConnectionString); Logger.LogInformation("Creating event processor host"); var host = new EventProcessorHost(eventProcessorHostName, builder.EntityPath, consumerGroupName, builder.ToString(), checkpointManager, leaseManager) { PartitionManagerOptions = new PartitionManagerOptions { RenewInterval = TimeSpan.FromSeconds(leaseRenewalSeconds), LeaseDuration = TimeSpan.FromSeconds(leaseDurationSeconds) } }; Logger.LogInformation("Initializing checkpoint manager"); checkpointManager.Initialize(host); Logger.LogInformation("Initializing lease manager"); await leaseManager.InitializeAsync(host).ConfigureAwait(true); Logger.LogInformation("Receive timeout is set to: {receiveTimeout} seconds", receiveTimeout); var processorOptions = new EventProcessorOptions { InitialOffsetProvider = (partitionId) => _startPosition, InvokeProcessorAfterReceiveTimeout = true, MaxBatchSize = maxBatchSize, PrefetchCount = prefetchCount, ReceiveTimeout = TimeSpan.FromSeconds(receiveTimeout) }; Logger.LogInformation("Registering event processor host"); await host.RegisterEventProcessorFactoryAsync(this, processorOptions).ConfigureAwait(false); Logger.LogInformation("Starting loop"); while (!ContainerCancellationToken.IsCancellationRequested) { await Task.WhenAny(Task.Delay(iterationDelay), ContainerTask).ConfigureAwait(false); _iterationCounter.Increment(); Logger.LogDebug("Derived Iteration completed"); } Logger.LogWarning("Unregistering event processory"); await host.UnregisterEventProcessorAsync().ConfigureAwait(true); // Service should be flagged as no longer available before any cleanup tasks IsContainerAvailable = false; Logger.LogWarning("Ending execution loop"); }
/// <summary> /// An asynchronous version of the entry point to enable asynchronous all the way down /// </summary> private static async Task MainAsync() { var processingManagerId = 0; var configuration = GetConfiguration(LogLevel.Warning, LogLevel.Information); var metricFactory = GetMetricFactory(); var processorOptions = new EventProcessorOptions { InitialOffsetProvider = (partitionId) => EventPosition.FromStart(), InvokeProcessorAfterReceiveTimeout = true, MaxBatchSize = 100, PrefetchCount = 300, ReceiveTimeout = TimeSpan.FromSeconds(15) }; var manager = new FixedPartitionManager(configuration.PartitionManagerLogger, configuration.EventHubConnectionString, processingManagerId); await manager.InitializeAsync(2).ConfigureAwait(true); var epochRecorder = new AzureStorageEpochRecorder(configuration.EpochRecorderLogger, metricFactory, configuration.ConsumerGroupName, configuration.CheckpointConnectionString, configuration.EpochContainerName, null); var leaseManager = new FixedLeaseManager(configuration.LeaseLogger, manager, epochRecorder); var checkpointManager = new AzureStorageCheckpointManager(configuration.CheckpointLogger, metricFactory, configuration.CheckpointConnectionString, configuration.CheckpointContainerName, null); var builder = new EventHubsConnectionStringBuilder(configuration.EventHubConnectionString); var host = new EventProcessorHost(configuration.EventProcessorHostName, builder.EntityPath, configuration.ConsumerGroupName, builder.ToString(), checkpointManager, leaseManager) { PartitionManagerOptions = new PartitionManagerOptions() }; host.PartitionManagerOptions.RenewInterval = TimeSpan.FromSeconds(10); host.PartitionManagerOptions.LeaseDuration = TimeSpan.FromSeconds(20); checkpointManager.Initialize(host); await leaseManager.InitializeAsync(host).ConfigureAwait(true); await host.RegisterEventProcessorAsync <SampleProcessor>(processorOptions).ConfigureAwait(true); string line; do { Console.WriteLine("Event Processor Started. To change the event processor manager count enter a number and press <ENTER> or quit to exit."); line = Console.ReadLine(); if (!string.IsNullOrEmpty(line)) { line = line.Trim(); } if (!string.IsNullOrEmpty(line)) { if (int.TryParse(line, out var managerCount)) { if (managerCount < 1 || managerCount > 1025) { Console.WriteLine("Valid numbers are 1 to 1025"); } else { await manager.UpdateManagerQuantityAsync(managerCount, CancellationToken.None).ConfigureAwait(true); } } else if (!string.Equals(line, "quit", StringComparison.OrdinalIgnoreCase)) { Console.WriteLine("Did not recognize entry please try again."); } } } while (!string.Equals(line, "quit", StringComparison.OrdinalIgnoreCase)); await host.UnregisterEventProcessorAsync().ConfigureAwait(true); }