public void DetermineOwnership(int id, int quantity)
        {
            var logger  = GetLogger("testing");
            var manager = new FixedPartitionManager(logger, Environment.GetEnvironmentVariable("iotc_eventhub"), id);


            manager.InitializeAsync(quantity).GetAwaiter().GetResult();
            var partitions = manager.GetPartitions();
            var ownership  = new Dictionary <string, bool>();


            var ownedPartitions = manager.GetOwnedPartitions();

            for (var index = 0; index < partitions.Length; index++)
            {
                var partitionId = partitions[index];
                var isOwner     = manager.IsOwner(partitionId);

                ownership.Add(partitionId, isOwner);
            }

            for (var index = 0; index < partitions.Length; index++)
            {
                var partitionId = partitions[index];
                var shouldOwn   = (index % quantity == id);

                Assert.IsTrue(ownership[partitionId] == shouldOwn);
            }
        }
        public void SimpleManagerTest()
        {
            var connectionString = Environment.GetEnvironmentVariable("ConnectionString");

            // Retrieve the partition information from the Event or IoT Hub
            var client = EventHubClient.CreateFromConnectionString(connectionString);

            client.RetryPolicy = new RetryExponential(TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(10), 10);
            var runtimeInformation = client.GetRuntimeInformationAsync().GetAwaiter().GetResult();
            var partitions         = runtimeInformation.PartitionIds.OrderBy(item => item).ToArray();

            var logger  = new LoggerFactory().CreateLogger("test");
            var manager = new FixedPartitionManager(logger, connectionString, 0);

            manager.InitializeAsync(2).GetAwaiter().GetResult();

            var managerPartitions      = manager.GetPartitions();
            var managerOwnedPartitions = manager.GetPartitions();
            var isOwned = manager.IsOwner("0");

            Assert.IsTrue(managerPartitions.Length == partitions.Length, "Partition counts should match");
            Assert.IsTrue(managerPartitions.Intersect(partitions).ToArray().Length == partitions.Length, "Intersecting partition should include all partitions");

            var partitionHalf = (int)(partitions.Length / 2) - 1;

            Assert.IsTrue(Math.Abs(managerOwnedPartitions.Length - partitionHalf) <= 1, "Owned partitions should be approximately half in case of odd numbers");
            Assert.IsTrue(Math.Abs(managerPartitions.Intersect(managerOwnedPartitions).ToArray().Length - partitionHalf) <= 1, "Intersecting owned partition should include all partitions");
            Assert.IsTrue(isOwned, "Ownership of partition 0 is expected based on the known algorithm. If the algorithm of modulo fails this needs adjusted");
        }
Exemple #3
0
        /// <summary>
        /// Initializes a new instance of the type
        /// </summary>
        /// <param name="logger">The logger to write debugging and diagnostics information to</param>
        /// <param name="metricFactory">A factory to create metric containers from</param>
        /// <param name="connectionString">The connection string to the Event Hub</param>
        /// <param name="managerId">The zero based index of the fixed partition manager</param>
        /// <param name="managerCount">The number of fixed partition managers that cooperating</param>
        public FixedLeaseManager(ILogger logger, IMetricFactory metricFactory, string connectionString, int managerId, int managerCount)
        {
            _partitionManager = new FixedPartitionManager(logger, connectionString, managerId);
            _managerCount     = managerCount;

            _updateManagerCounter  = metricFactory.CreateCounter("flm-update-counter", "The number of times the manager count has been updated", false, new string[0]);
            _ownershipClaimCounter = metricFactory.CreateCounter("flm-ownership-claim-counter", "The number of times a partition claim request has been executed (multiples per request)", false, new string[0]);
            _listOwnershipCounter  = metricFactory.CreateCounter("flm-ownership-list-counter", "The number of times a partition claim list request has been executed", false, new string[0]);
        }
        /// <inheritdoc />
        protected sealed override async Task ExecuteAsync(CancellationToken cancellationToken)
        {
            Logger.LogInformation("Started execution loop");

            // Once the initial tasks are completed indicate the service is now available
            IsContainerAvailable = true;

            var checkpointStoreRoot      = Environment.GetEnvironmentVariable("CheckpointStore");
            var eventHubConnectionString = await GetEventHubConnectionStringAsync().ConfigureAwait(false);

            var checkpointConnectionString = await GetStorageConnectionStringAsync().ConfigureAwait(false);

            var consumerGroupName       = Environment.GetEnvironmentVariable("ConsumerGroup");
            var checkpointContainerName = Environment.GetEnvironmentVariable("CheckpointContainerName");
            var epochContainerName      = Environment.GetEnvironmentVariable("EpochContainerName");
            var eventProcessorHostName  = Environment.GetEnvironmentVariable("EventProcessorHostName");


            if (!int.TryParse(Environment.GetEnvironmentVariable("LeaseDurationSeconds"), out int leaseDurationSeconds) || leaseDurationSeconds < 20)
            {
                leaseDurationSeconds = 20;
            }
            if (!int.TryParse(Environment.GetEnvironmentVariable("LeaseRenewalSeconds"), out int leaseRenewalSeconds) || leaseRenewalSeconds < 10 || leaseRenewalSeconds >= leaseDurationSeconds)
            {
                leaseRenewalSeconds = 10;
            }
            if (!int.TryParse(Environment.GetEnvironmentVariable("ReceiveTimeoutSeconds"), out int receiveTimeout) || receiveTimeout < 5)
            {
                receiveTimeout = 60;
            }
            if (!int.TryParse(Environment.GetEnvironmentVariable("PrefetchCount"), out int prefetchCount) || prefetchCount < 0)
            {
                prefetchCount = 0;
            }
            if (!int.TryParse(Environment.GetEnvironmentVariable("MaxBatchSize"), out int maxBatchSize) || maxBatchSize < 1)
            {
                maxBatchSize = 1;
            }
            if (!int.TryParse(Environment.GetEnvironmentVariable("IterationSeconds"), out int iterationSeconds) || iterationSeconds < 1)
            {
                iterationSeconds = 1;
            }

            var iterationDelay = TimeSpan.FromSeconds(iterationSeconds);

            Logger.LogInformation("Creating fixed partition manager");
            _partitionManager = new FixedPartitionManager(_loggerFactory.CreateLogger(PartitionManagerLoggerName), eventHubConnectionString, KubernetesStatefulSetIndex.Value);

            Logger.LogInformation("Initializing fixed partition manager");
            await _partitionManager.InitializeAsync(KubernetesDesiredReplicaCount.Value).ConfigureAwait(true);

            Logger.LogInformation("Creating epoch reader");
            var epochRecorder = new AzureStorageEpochRecorder(_loggerFactory.CreateLogger(EpochRecorderLoggerName), MetricFactory, consumerGroupName, checkpointConnectionString, epochContainerName, null);

            Logger.LogInformation("Creating fixed lease manager");
            var leaseManager = new FixedLeaseManager(_loggerFactory.CreateLogger(LeaseManagerLoggerName), _partitionManager, epochRecorder);

            Logger.LogInformation("Creating Azure storage checkpoint manager");
            var checkpointManager = new AzureStorageCheckpointManager(_loggerFactory.CreateLogger(CheckpointManagerLoggerName), MetricFactory, checkpointConnectionString, checkpointContainerName, null);

            Logger.LogInformation("Building connection string");
            var builder = new EventHubsConnectionStringBuilder(eventHubConnectionString);

            Logger.LogInformation("Creating event processor host");
            var host = new EventProcessorHost(eventProcessorHostName, builder.EntityPath, consumerGroupName, builder.ToString(), checkpointManager, leaseManager)
            {
                PartitionManagerOptions = new PartitionManagerOptions
                {
                    RenewInterval = TimeSpan.FromSeconds(leaseRenewalSeconds),
                    LeaseDuration = TimeSpan.FromSeconds(leaseDurationSeconds)
                }
            };

            Logger.LogInformation("Initializing checkpoint manager");
            checkpointManager.Initialize(host);

            Logger.LogInformation("Initializing lease manager");
            await leaseManager.InitializeAsync(host).ConfigureAwait(true);

            Logger.LogInformation("Receive timeout is set to: {receiveTimeout} seconds", receiveTimeout);

            var processorOptions = new EventProcessorOptions
            {
                InitialOffsetProvider = (partitionId) => _startPosition,
                InvokeProcessorAfterReceiveTimeout = true,
                MaxBatchSize   = maxBatchSize,
                PrefetchCount  = prefetchCount,
                ReceiveTimeout = TimeSpan.FromSeconds(receiveTimeout)
            };

            Logger.LogInformation("Registering event processor host");

            await host.RegisterEventProcessorFactoryAsync(this, processorOptions).ConfigureAwait(false);

            Logger.LogInformation("Starting loop");

            while (!ContainerCancellationToken.IsCancellationRequested)
            {
                await Task.WhenAny(Task.Delay(iterationDelay), ContainerTask).ConfigureAwait(false);

                _iterationCounter.Increment();
                Logger.LogDebug("Derived Iteration completed");
            }

            Logger.LogWarning("Unregistering event processory");
            await host.UnregisterEventProcessorAsync().ConfigureAwait(true);


            // Service should be flagged as no longer available before any cleanup tasks
            IsContainerAvailable = false;

            Logger.LogWarning("Ending execution loop");
        }
Exemple #5
0
        /// <summary>
        /// An asynchronous version of the entry point to enable asynchronous all the way down
        /// </summary>
        private static async Task MainAsync()
        {
            var processingManagerId = 0;

            var configuration = GetConfiguration(LogLevel.Warning, LogLevel.Information);
            var metricFactory = GetMetricFactory();

            var processorOptions = new EventProcessorOptions
            {
                InitialOffsetProvider = (partitionId) => EventPosition.FromStart(),
                InvokeProcessorAfterReceiveTimeout = true,
                MaxBatchSize   = 100,
                PrefetchCount  = 300,
                ReceiveTimeout = TimeSpan.FromSeconds(15)
            };

            var manager = new FixedPartitionManager(configuration.PartitionManagerLogger, configuration.EventHubConnectionString, processingManagerId);

            await manager.InitializeAsync(2).ConfigureAwait(true);

            var epochRecorder = new AzureStorageEpochRecorder(configuration.EpochRecorderLogger, metricFactory, configuration.ConsumerGroupName, configuration.CheckpointConnectionString, configuration.EpochContainerName, null);

            var leaseManager      = new FixedLeaseManager(configuration.LeaseLogger, manager, epochRecorder);
            var checkpointManager = new AzureStorageCheckpointManager(configuration.CheckpointLogger, metricFactory, configuration.CheckpointConnectionString, configuration.CheckpointContainerName, null);

            var builder = new EventHubsConnectionStringBuilder(configuration.EventHubConnectionString);

            var host = new EventProcessorHost(configuration.EventProcessorHostName, builder.EntityPath, configuration.ConsumerGroupName, builder.ToString(), checkpointManager, leaseManager)
            {
                PartitionManagerOptions = new PartitionManagerOptions()
            };

            host.PartitionManagerOptions.RenewInterval = TimeSpan.FromSeconds(10);
            host.PartitionManagerOptions.LeaseDuration = TimeSpan.FromSeconds(20);

            checkpointManager.Initialize(host);
            await leaseManager.InitializeAsync(host).ConfigureAwait(true);

            await host.RegisterEventProcessorAsync <SampleProcessor>(processorOptions).ConfigureAwait(true);

            string line;

            do
            {
                Console.WriteLine("Event Processor Started. To change the event processor manager count enter a number and press <ENTER> or quit to exit.");
                line = Console.ReadLine();

                if (!string.IsNullOrEmpty(line))
                {
                    line = line.Trim();
                }

                if (!string.IsNullOrEmpty(line))
                {
                    if (int.TryParse(line, out var managerCount))
                    {
                        if (managerCount < 1 || managerCount > 1025)
                        {
                            Console.WriteLine("Valid numbers are 1 to 1025");
                        }
                        else
                        {
                            await manager.UpdateManagerQuantityAsync(managerCount, CancellationToken.None).ConfigureAwait(true);
                        }
                    }
                    else if (!string.Equals(line, "quit", StringComparison.OrdinalIgnoreCase))
                    {
                        Console.WriteLine("Did not recognize entry please try again.");
                    }
                }
            } while (!string.Equals(line, "quit", StringComparison.OrdinalIgnoreCase));

            await host.UnregisterEventProcessorAsync().ConfigureAwait(true);
        }