public void DetermineOwnership(int id, int quantity)
{
var logger = GetLogger("testing");
var manager = new FixedPartitionManager(logger, Environment.GetEnvironmentVariable("iotc_eventhub"), id);
manager.InitializeAsync(quantity).GetAwaiter().GetResult();
var partitions = manager.GetPartitions();
var ownership = new Dictionary <string, bool>();
var ownedPartitions = manager.GetOwnedPartitions();
for (var index = 0; index < partitions.Length; index++)
{
var partitionId = partitions[index];
var isOwner = manager.IsOwner(partitionId);
ownership.Add(partitionId, isOwner);
}
for (var index = 0; index < partitions.Length; index++)
{
var partitionId = partitions[index];
var shouldOwn = (index % quantity == id);
Assert.IsTrue(ownership[partitionId] == shouldOwn);
}
}
public void SimpleManagerTest()
{
var connectionString = Environment.GetEnvironmentVariable("ConnectionString");
// Retrieve the partition information from the Event or IoT Hub
var client = EventHubClient.CreateFromConnectionString(connectionString);
client.RetryPolicy = new RetryExponential(TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(10), 10);
var runtimeInformation = client.GetRuntimeInformationAsync().GetAwaiter().GetResult();
var partitions = runtimeInformation.PartitionIds.OrderBy(item => item).ToArray();
var logger = new LoggerFactory().CreateLogger("test");
var manager = new FixedPartitionManager(logger, connectionString, 0);
manager.InitializeAsync(2).GetAwaiter().GetResult();
var managerPartitions = manager.GetPartitions();
var managerOwnedPartitions = manager.GetPartitions();
var isOwned = manager.IsOwner("0");
Assert.IsTrue(managerPartitions.Length == partitions.Length, "Partition counts should match");
Assert.IsTrue(managerPartitions.Intersect(partitions).ToArray().Length == partitions.Length, "Intersecting partition should include all partitions");
var partitionHalf = (int)(partitions.Length / 2) - 1;
Assert.IsTrue(Math.Abs(managerOwnedPartitions.Length - partitionHalf) <= 1, "Owned partitions should be approximately half in case of odd numbers");
Assert.IsTrue(Math.Abs(managerPartitions.Intersect(managerOwnedPartitions).ToArray().Length - partitionHalf) <= 1, "Intersecting owned partition should include all partitions");
Assert.IsTrue(isOwned, "Ownership of partition 0 is expected based on the known algorithm. If the algorithm of modulo fails this needs adjusted");
}
/// <summary>
/// Initializes a new instance of the type
/// </summary>
/// <param name="logger">The logger to write debugging and diagnostics information to</param>
/// <param name="metricFactory">A factory to create metric containers from</param>
/// <param name="connectionString">The connection string to the Event Hub</param>
/// <param name="managerId">The zero based index of the fixed partition manager</param>
/// <param name="managerCount">The number of fixed partition managers that cooperating</param>
public FixedLeaseManager(ILogger logger, IMetricFactory metricFactory, string connectionString, int managerId, int managerCount)
{
_partitionManager = new FixedPartitionManager(logger, connectionString, managerId);
_managerCount = managerCount;
_updateManagerCounter = metricFactory.CreateCounter("flm-update-counter", "The number of times the manager count has been updated", false, new string[0]);
_ownershipClaimCounter = metricFactory.CreateCounter("flm-ownership-claim-counter", "The number of times a partition claim request has been executed (multiples per request)", false, new string[0]);
_listOwnershipCounter = metricFactory.CreateCounter("flm-ownership-list-counter", "The number of times a partition claim list request has been executed", false, new string[0]);
}
/// <inheritdoc />
protected sealed override async Task ExecuteAsync(CancellationToken cancellationToken)
{
Logger.LogInformation("Started execution loop");
// Once the initial tasks are completed indicate the service is now available
IsContainerAvailable = true;
var checkpointStoreRoot = Environment.GetEnvironmentVariable("CheckpointStore");
var eventHubConnectionString = await GetEventHubConnectionStringAsync().ConfigureAwait(false);
var checkpointConnectionString = await GetStorageConnectionStringAsync().ConfigureAwait(false);
var consumerGroupName = Environment.GetEnvironmentVariable("ConsumerGroup");
var checkpointContainerName = Environment.GetEnvironmentVariable("CheckpointContainerName");
var epochContainerName = Environment.GetEnvironmentVariable("EpochContainerName");
var eventProcessorHostName = Environment.GetEnvironmentVariable("EventProcessorHostName");
if (!int.TryParse(Environment.GetEnvironmentVariable("LeaseDurationSeconds"), out int leaseDurationSeconds) || leaseDurationSeconds < 20)
{
leaseDurationSeconds = 20;
}
if (!int.TryParse(Environment.GetEnvironmentVariable("LeaseRenewalSeconds"), out int leaseRenewalSeconds) || leaseRenewalSeconds < 10 || leaseRenewalSeconds >= leaseDurationSeconds)
{
leaseRenewalSeconds = 10;
}
if (!int.TryParse(Environment.GetEnvironmentVariable("ReceiveTimeoutSeconds"), out int receiveTimeout) || receiveTimeout < 5)
{
receiveTimeout = 60;
}
if (!int.TryParse(Environment.GetEnvironmentVariable("PrefetchCount"), out int prefetchCount) || prefetchCount < 0)
{
prefetchCount = 0;
}
if (!int.TryParse(Environment.GetEnvironmentVariable("MaxBatchSize"), out int maxBatchSize) || maxBatchSize < 1)
{
maxBatchSize = 1;
}
if (!int.TryParse(Environment.GetEnvironmentVariable("IterationSeconds"), out int iterationSeconds) || iterationSeconds < 1)
{
iterationSeconds = 1;
}
var iterationDelay = TimeSpan.FromSeconds(iterationSeconds);
Logger.LogInformation("Creating fixed partition manager");
_partitionManager = new FixedPartitionManager(_loggerFactory.CreateLogger(PartitionManagerLoggerName), eventHubConnectionString, KubernetesStatefulSetIndex.Value);
Logger.LogInformation("Initializing fixed partition manager");
await _partitionManager.InitializeAsync(KubernetesDesiredReplicaCount.Value).ConfigureAwait(true);
Logger.LogInformation("Creating epoch reader");
var epochRecorder = new AzureStorageEpochRecorder(_loggerFactory.CreateLogger(EpochRecorderLoggerName), MetricFactory, consumerGroupName, checkpointConnectionString, epochContainerName, null);
Logger.LogInformation("Creating fixed lease manager");
var leaseManager = new FixedLeaseManager(_loggerFactory.CreateLogger(LeaseManagerLoggerName), _partitionManager, epochRecorder);
Logger.LogInformation("Creating Azure storage checkpoint manager");
var checkpointManager = new AzureStorageCheckpointManager(_loggerFactory.CreateLogger(CheckpointManagerLoggerName), MetricFactory, checkpointConnectionString, checkpointContainerName, null);
Logger.LogInformation("Building connection string");
var builder = new EventHubsConnectionStringBuilder(eventHubConnectionString);
Logger.LogInformation("Creating event processor host");
var host = new EventProcessorHost(eventProcessorHostName, builder.EntityPath, consumerGroupName, builder.ToString(), checkpointManager, leaseManager)
{
PartitionManagerOptions = new PartitionManagerOptions
{
RenewInterval = TimeSpan.FromSeconds(leaseRenewalSeconds),
LeaseDuration = TimeSpan.FromSeconds(leaseDurationSeconds)
}
};
Logger.LogInformation("Initializing checkpoint manager");
checkpointManager.Initialize(host);
Logger.LogInformation("Initializing lease manager");
await leaseManager.InitializeAsync(host).ConfigureAwait(true);
Logger.LogInformation("Receive timeout is set to: {receiveTimeout} seconds", receiveTimeout);
var processorOptions = new EventProcessorOptions
{
InitialOffsetProvider = (partitionId) => _startPosition,
InvokeProcessorAfterReceiveTimeout = true,
MaxBatchSize = maxBatchSize,
PrefetchCount = prefetchCount,
ReceiveTimeout = TimeSpan.FromSeconds(receiveTimeout)
};
Logger.LogInformation("Registering event processor host");
await host.RegisterEventProcessorFactoryAsync(this, processorOptions).ConfigureAwait(false);
Logger.LogInformation("Starting loop");
while (!ContainerCancellationToken.IsCancellationRequested)
{
await Task.WhenAny(Task.Delay(iterationDelay), ContainerTask).ConfigureAwait(false);
_iterationCounter.Increment();
Logger.LogDebug("Derived Iteration completed");
}
Logger.LogWarning("Unregistering event processory");
await host.UnregisterEventProcessorAsync().ConfigureAwait(true);
// Service should be flagged as no longer available before any cleanup tasks
IsContainerAvailable = false;
Logger.LogWarning("Ending execution loop");
}
/// <summary>
/// An asynchronous version of the entry point to enable asynchronous all the way down
/// </summary>
private static async Task MainAsync()
{
var processingManagerId = 0;
var configuration = GetConfiguration(LogLevel.Warning, LogLevel.Information);
var metricFactory = GetMetricFactory();
var processorOptions = new EventProcessorOptions
{
InitialOffsetProvider = (partitionId) => EventPosition.FromStart(),
InvokeProcessorAfterReceiveTimeout = true,
MaxBatchSize = 100,
PrefetchCount = 300,
ReceiveTimeout = TimeSpan.FromSeconds(15)
};
var manager = new FixedPartitionManager(configuration.PartitionManagerLogger, configuration.EventHubConnectionString, processingManagerId);
await manager.InitializeAsync(2).ConfigureAwait(true);
var epochRecorder = new AzureStorageEpochRecorder(configuration.EpochRecorderLogger, metricFactory, configuration.ConsumerGroupName, configuration.CheckpointConnectionString, configuration.EpochContainerName, null);
var leaseManager = new FixedLeaseManager(configuration.LeaseLogger, manager, epochRecorder);
var checkpointManager = new AzureStorageCheckpointManager(configuration.CheckpointLogger, metricFactory, configuration.CheckpointConnectionString, configuration.CheckpointContainerName, null);
var builder = new EventHubsConnectionStringBuilder(configuration.EventHubConnectionString);
var host = new EventProcessorHost(configuration.EventProcessorHostName, builder.EntityPath, configuration.ConsumerGroupName, builder.ToString(), checkpointManager, leaseManager)
{
PartitionManagerOptions = new PartitionManagerOptions()
};
host.PartitionManagerOptions.RenewInterval = TimeSpan.FromSeconds(10);
host.PartitionManagerOptions.LeaseDuration = TimeSpan.FromSeconds(20);
checkpointManager.Initialize(host);
await leaseManager.InitializeAsync(host).ConfigureAwait(true);
await host.RegisterEventProcessorAsync <SampleProcessor>(processorOptions).ConfigureAwait(true);
string line;
do
{
Console.WriteLine("Event Processor Started. To change the event processor manager count enter a number and press <ENTER> or quit to exit.");
line = Console.ReadLine();
if (!string.IsNullOrEmpty(line))
{
line = line.Trim();
}
if (!string.IsNullOrEmpty(line))
{
if (int.TryParse(line, out var managerCount))
{
if (managerCount < 1 || managerCount > 1025)
{
Console.WriteLine("Valid numbers are 1 to 1025");
}
else
{
await manager.UpdateManagerQuantityAsync(managerCount, CancellationToken.None).ConfigureAwait(true);
}
}
else if (!string.Equals(line, "quit", StringComparison.OrdinalIgnoreCase))
{
Console.WriteLine("Did not recognize entry please try again.");
}
}
} while (!string.Equals(line, "quit", StringComparison.OrdinalIgnoreCase));
await host.UnregisterEventProcessorAsync().ConfigureAwait(true);
}
