/// <summary>
/// Create a new host to process events from an Event Hub with provided <see cref="TokenProvider"/>
/// </summary>
/// <param name="hostName">Name of the processor host. MUST BE UNIQUE. Strongly recommend including a Guid to ensure uniqueness.</param>
/// <param name="endpointAddress">Fully qualified domain name for Event Hubs. Most likely, {yournamespace}.servicebus.windows.net</param>
/// <param name="eventHubPath">The name of the EventHub.</param>
/// <param name="consumerGroupName">The name of the consumer group within the Event Hub.</param>
/// <param name="tokenProvider">Token provider which will generate security tokens for authorization.</param>
/// <param name="checkpointManager">Object implementing ICheckpointManager which handles partition checkpointing.</param>
/// <param name="leaseManager">Object implementing ILeaseManager which handles leases for partitions.</param>
/// <param name="operationTimeout">Operation timeout for Event Hubs operations.</param>
/// <param name="transportType">Transport type on connection.</param>
public EventProcessorHost(
string hostName,
Uri endpointAddress,
string eventHubPath,
string consumerGroupName,
ITokenProvider tokenProvider,
ICheckpointManager checkpointManager,
ILeaseManager leaseManager,
TimeSpan?operationTimeout = null,
TransportType transportType = TransportType.Amqp)
{
Guard.ArgumentNotNullOrWhiteSpace(nameof(hostName), hostName);
Guard.ArgumentNotNull(nameof(endpointAddress), endpointAddress);
Guard.ArgumentNotNullOrWhiteSpace(nameof(eventHubPath), eventHubPath);
Guard.ArgumentNotNullOrWhiteSpace(nameof(consumerGroupName), consumerGroupName);
Guard.ArgumentNotNull(nameof(tokenProvider), tokenProvider);
Guard.ArgumentNotNull(nameof(checkpointManager), checkpointManager);
Guard.ArgumentNotNull(nameof(leaseManager), leaseManager);
this.HostName = hostName;
this.EndpointAddress = endpointAddress;
this.EventHubPath = eventHubPath;
this.ConsumerGroupName = consumerGroupName;
this.tokenProvider = tokenProvider;
this.CheckpointManager = checkpointManager;
this.LeaseManager = leaseManager;
this.TransportType = transportType;
this.OperationTimeout = operationTimeout ?? ClientConstants.DefaultOperationTimeout;
this.PartitionManager = new PartitionManager(this);
ProcessorEventSource.Log.EventProcessorHostCreated(this.HostName, this.EventHubPath);
}
public Task RegisterProcessorFactoryAsync(Lease lease, ICheckpointManager checkpointManager, IEventProcessorFactory eventProcessorFactory, EventProcessorOptions processorOptions)
{
base.ThrowIfDisposed();
if (lease == null)
{
throw Fx.Exception.ArgumentNull("lease");
}
if (checkpointManager == null)
{
throw Fx.Exception.ArgumentNull("checkpointManager");
}
if (eventProcessorFactory == null)
{
throw Fx.Exception.ArgumentNull("eventProcessorFactory");
}
if (processorOptions == null)
{
throw Fx.Exception.ArgumentNull("processorOptions");
}
EventProcessorLifecycleManager eventProcessorLifecycleManager = null;
if (this.handlers.TryGetValue(lease, out eventProcessorLifecycleManager))
{
ExceptionTrace exception = Fx.Exception;
string eventProcessorAlreadyRegistered = Resources.EventProcessorAlreadyRegistered;
object[] partitionId = new object[] { lease.PartitionId };
throw exception.AsError(new InvalidOperationException(Microsoft.ServiceBus.SR.GetString(eventProcessorAlreadyRegistered, partitionId)), null);
}
eventProcessorLifecycleManager = new EventProcessorLifecycleManager(this, lease, checkpointManager, processorOptions);
if (!this.handlers.TryAdd(lease, eventProcessorLifecycleManager))
{
return(TaskHelpers.GetCompletedTask <object>(null));
}
return(eventProcessorLifecycleManager.RegisterProcessorFactoryAsync(eventProcessorFactory));
}
protected CatchUpObserver(ICheckpointManager checkpointManager, IReplayManager replayManager,
GetGlobalCheckpoint getGlobalCheckpoint)
{
_checkpointManager = checkpointManager ?? throw new ArgumentNullException(nameof(checkpointManager));
_replayManager = replayManager ?? throw new ArgumentNullException(nameof(replayManager));
_getGlobalCheckpoint = getGlobalCheckpoint ?? throw new ArgumentNullException(nameof(getGlobalCheckpoint));
}
public EventProcessorLifecycleManager(EventHubConsumerGroup eventHubConsumer, Lease lease, ICheckpointManager checkpointManager, EventProcessorOptions processorOptions)
{
this.eventHubConsumer = eventHubConsumer;
this.lease = lease;
this.checkpointManager = checkpointManager;
this.processorOptions = processorOptions;
}
public PostgresCheckpointManagerTests()
{
var fixture = new Fixture();
_options = fixture.Build <PostgresOptions>()
.Without(o => o.Host)
.Without(o => o.Port)
.Without(o => o.Database)
.Without(o => o.Username)
.Without(o => o.Password)
.Without(o => o.Pooling)
.Without(o => o.Schema)
.Do(o =>
{
o.Host = Environment.GetEnvironmentVariable("POSTGRES_HOST") ?? "localhost";
o.Port = Convert.ToInt32(Environment.GetEnvironmentVariable("POSTGRES_PORT") ?? "5432");
o.Database = Environment.GetEnvironmentVariable("POSTGRES_DATABASE") ?? "postgres";
o.Username = Environment.GetEnvironmentVariable("POSTGRES_USERNAME") ?? "postgres";
o.Password = Environment.GetEnvironmentVariable("POSTGRES_PASSWORD") ?? "postgres";
o.Pooling = Convert.ToBoolean(Environment.GetEnvironmentVariable("POSTGRES_POOLING") ?? "False");
o.Schema = Environment.GetEnvironmentVariable("POSTGRES_SCHEMA") ?? "public";
})
.Create();
_connection = new PostgresConnection(_options);
_sut = new PostgresCheckpointManager(_options, _connection);
}
async Task InitializeAsync()
{
this.documentClient = new DocumentClient(this.collectionLocation.Uri, this.collectionLocation.MasterKey, this.collectionLocation.ConnectionPolicy);
Uri databaseUri = UriFactory.CreateDatabaseUri(this.collectionLocation.DatabaseName);
Database database = await this.documentClient.ReadDatabaseAsync(databaseUri);
Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.collectionLocation.DatabaseName, this.collectionLocation.CollectionName);
ResourceResponse <DocumentCollection> collectionResponse = await this.documentClient.ReadDocumentCollectionAsync(
collectionUri,
new RequestOptions { PopulateQuotaInfo = true });
DocumentCollection collection = collectionResponse.Resource;
this.collectionSelfLink = collection.SelfLink;
// Grab the options-supplied prefix if present otherwise leave it empty.
string optionsPrefix = this.options.LeasePrefix ?? string.Empty;
// Beyond this point all access to collection is done via this self link: if collection is removed, we won't access new one using same name by accident.
this.leasePrefix = string.Format(CultureInfo.InvariantCulture, "{0}{1}_{2}_{3}", optionsPrefix, this.collectionLocation.Uri.Host, database.ResourceId, collection.ResourceId);
var leaseManager = new DocumentServiceLeaseManager(
this.auxCollectionLocation,
this.leasePrefix,
this.options.LeaseExpirationInterval,
this.options.LeaseRenewInterval);
await leaseManager.InitializeAsync();
this.leaseManager = leaseManager;
this.checkpointManager = (ICheckpointManager)leaseManager;
if (this.options.DiscardExistingLeases)
{
TraceLog.Warning(string.Format("Host '{0}': removing all leases, as requested by ChangeFeedHostOptions", this.HostName));
await this.leaseManager.DeleteAllAsync();
}
// Note: lease store is never stale as we use monitored colleciton Rid as id prefix for aux collection.
// Collection was removed and re-created, the rid would change.
// If it's not deleted, it's not stale. If it's deleted, it's not stale as it doesn't exist.
await this.leaseManager.CreateLeaseStoreIfNotExistsAsync();
var ranges = new Dictionary <string, PartitionKeyRange>();
foreach (var range in await this.EnumPartitionKeyRangesAsync(this.collectionSelfLink))
{
ranges.Add(range.Id, range);
}
TraceLog.Informational(string.Format("Source collection: '{0}', {1} partition(s), {2} document(s)", this.collectionLocation.CollectionName, ranges.Count, GetDocumentCount(collectionResponse)));
await this.CreateLeases(ranges);
this.partitionManager = new PartitionManager <DocumentServiceLease>(this.HostName, this.leaseManager, this.options);
await this.partitionManager.SubscribeAsync(this);
await this.partitionManager.InitializeAsync();
}
protected DomainEventsProcessor(ICheckpointManager checkpointManager)
{
Guard.ArgumentNotNull(checkpointManager, nameof(checkpointManager));
CheckpointManager = checkpointManager;
var handler = this as IDomainEventsHandler;
_handler = handler ?? throw new InvalidOperationException("Processor should implement IDomainEventsHandler inteface to be used as Handler");
}
/// <summary>
/// Initializes a new instance of the type
/// </summary>
/// <param name="logger">The logger to write debugging and diagnostics information to</param>
/// <param name="metricFactory">The factory to create metrics containers from.</param>
/// <param name="partitionId">The identifier of the partition that the context represents.</param>
/// <param name="readLastEnqueuedEventProperties">A function that can be used to read the last enqueued event properties for the partition.</param>
/// <param name="checkpointManager">The checkpoint manager to used to create the checkpoint</param>
internal ProcessorPartitionContext(ILogger logger, IMetricFactory metricFactory, string partitionId, LastPropertiesReader readLastEnqueuedEventProperties, ICheckpointManager checkpointManager) : base(partitionId)
{
_logger = logger;
_checkpointManager = checkpointManager;
_readLastEnqueuedEventProperties = readLastEnqueuedEventProperties;
_checkpointCounter = metricFactory.CreateCounter($"ppc-checkpoint-counter-{partitionId}", $"The number of times that the checkpoint has been called for partition {partitionId}", false, new string[0]);
_checkpointTiming = metricFactory.CreateSummary($"ppc-checkpoint-timing-{partitionId}", $"The time taken to perform checkpoints for partition {partitionId}", 10, false, new string[0]);
}
public async ValueTask LocalDeviceSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync, [Values] bool testCommitCookie)
{
checkpointManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme($"{TestUtils.MethodTestDir}/{TestUtils.AzureTestDirectory}"));
await SimpleRecoveryTest1_Worker(checkpointType, isAsync, testCommitCookie);
checkpointManager.PurgeAll();
}
public async ValueTask PageBlobSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync, [Values] bool testCommitCookie)
{
TestUtils.IgnoreIfNotRunningAzureTests();
checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await SimpleRecoveryTest1_Worker(checkpointType, isAsync, testCommitCookie);
checkpointManager.PurgeAll();
}
public void Recover(Guid guid, ICheckpointManager checkpointManager)
{
var metadata = checkpointManager.GetIndexCommitMetadata(guid);
if (metadata == null)
{
throw new FasterException("Invalid index commit metadata for ID " + guid.ToString());
}
Initialize(new StreamReader(new MemoryStream(metadata)));
}
/// <summary>
/// Create host wrapper
/// </summary>
/// <param name="factory"></param>
/// <param name="config"></param>
/// <param name="logger"></param>
/// <param name="checkpoint"></param>
/// <param name="lease"></param>
public EventProcessorHost(IEventProcessorFactory factory,
IEventProcessorConfig config, ICheckpointManager checkpoint,
ILeaseManager lease, ILogger logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_config = config ?? throw new ArgumentNullException(nameof(config));
_factory = factory ?? throw new ArgumentNullException(nameof(factory));
_lease = lease;
_checkpoint = checkpoint;
_lock = new SemaphoreSlim(1);
}
public void Setup()
{
TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true);
checkpointManager = default;
checkpointDir = default;
inputArray = new AdId[numOps];
for (int i = 0; i < numOps; i++)
{
inputArray[i].adId = i;
}
}
/// <summary>
/// Recover info from token
/// </summary>
/// <param name="token"></param>
/// <param name="checkpointManager"></param>
/// <returns></returns>
internal void Recover(Guid token, ICheckpointManager checkpointManager)
{
var metadata = checkpointManager.GetLogCommitMetadata(token);
if (metadata == null)
{
throw new FasterException("Invalid log commit metadata for ID " + token.ToString());
}
Initialize(new StreamReader(new MemoryStream(metadata)));
}
/// <summary>
/// Create a new host to process events from an Event Hub.
///
/// <para>This overload of the constructor allows maximum flexibility.
/// This one allows the caller to specify the name of the processor host as well.
/// The overload also allows the caller to provide their own lease and checkpoint managers to replace the built-in
/// ones based on Azure Storage.</para>
/// </summary>
/// <param name="hostName">Name of the processor host. MUST BE UNIQUE. Strongly recommend including a Guid to ensure uniqueness.</param>
/// <param name="eventHubPath">The name of the EventHub.</param>
/// <param name="consumerGroupName">The name of the consumer group within the Event Hub.</param>
/// <param name="eventHubConnectionString">Connection string for the Event Hub to receive from.</param>
/// <param name="checkpointManager">Object implementing ICheckpointManager which handles partition checkpointing.</param>
/// <param name="leaseManager">Object implementing ILeaseManager which handles leases for partitions.</param>
public EventProcessorHost(
string hostName,
string eventHubPath,
string consumerGroupName,
string eventHubConnectionString,
ICheckpointManager checkpointManager,
ILeaseManager leaseManager)
{
if (string.IsNullOrEmpty(consumerGroupName))
{
throw new ArgumentNullException(nameof(consumerGroupName));
}
else if (checkpointManager == null || leaseManager == null)
{
throw new ArgumentNullException(checkpointManager == null ? nameof(checkpointManager) : nameof(leaseManager));
}
var csb = new EventHubsConnectionStringBuilder(eventHubConnectionString);
if (string.IsNullOrEmpty(eventHubPath))
{
// Entity path is expected in the connection string if not provided with eventHubPath parameter.
if (string.IsNullOrEmpty(csb.EntityPath))
{
throw new ArgumentException(nameof(eventHubConnectionString),
"Provide EventHub entity path either in eventHubPath parameter or in eventHubConnectionString.");
}
}
else
{
// Entity path should not conflict with connection string.
if (!string.IsNullOrEmpty(csb.EntityPath) &&
string.Compare(csb.EntityPath, eventHubPath, StringComparison.OrdinalIgnoreCase) != 0)
{
throw new ArgumentException(nameof(eventHubConnectionString),
"Provided EventHub path in eventHubPath parameter conflicts with the path in provided EventHubs connection string.");
}
csb.EntityPath = eventHubPath;
}
this.HostName = hostName;
this.EventHubPath = csb.EntityPath;
this.ConsumerGroupName = consumerGroupName;
this.eventHubConnectionString = csb.ToString();
this.CheckpointManager = checkpointManager;
this.LeaseManager = leaseManager;
this.TransportType = csb.TransportType;
this.OperationTimeout = csb.OperationTimeout;
this.EndpointAddress = csb.Endpoint;
this.PartitionManager = new PartitionManager(this);
ProcessorEventSource.Log.EventProcessorHostCreated(this.HostName, this.EventHubPath);
}
public DomainEventsProcessor(IDomainEventsHandler handler, ICheckpointManager checkpointManager, IDomainEventsFaultHandler faultHandler = null)
{
Guard.ArgumentNotNull(handler, nameof(handler));
Guard.ArgumentNotNull(checkpointManager, nameof(checkpointManager));
if (faultHandler == null)
{
faultHandler = new DoNothingFaultHandler();
}
_handler = handler;
_faultHandler = faultHandler;
CheckpointManager = checkpointManager; // ?? new InMemoryCheckpointManager();
}
async Task InitializeAsync()
{
this.documentClient = new DocumentClient(this.collectionLocation.Uri, this.collectionLocation.MasterKey, this.collectionLocation.ConnectionPolicy);
Uri databaseUri = UriFactory.CreateDatabaseUri(this.collectionLocation.DatabaseName);
Database database = await this.documentClient.ReadDatabaseAsync(databaseUri);
Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.collectionLocation.DatabaseName, this.collectionLocation.CollectionName);
DocumentCollection collection = await this.documentClient.ReadDocumentCollectionAsync(collectionUri);
this.collectionSelfLink = collection.SelfLink;
// Beyond this point all access to colleciton is done via this self link: if collection is removed, we won't access new one using same name by accident.
this.leasePrefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_{2}", this.collectionLocation.Uri.Host, database.ResourceId, collection.ResourceId);
var leaseManager = new DocumentServiceLeaseManager(
this.auxCollectionLocation,
this.leasePrefix,
this.options.LeaseExpirationInterval,
this.options.LeaseRenewInterval);
await leaseManager.InitializeAsync();
this.leaseManager = leaseManager;
this.checkpointManager = (ICheckpointManager)leaseManager;
if (this.options.DiscardExistingLeases)
{
TraceLog.Warning(string.Format("Host '{0}': removing all leases, as requested by ChangeFeedHostOptions", this.HostName));
await this.leaseManager.DeleteAllAsync();
}
// Note: lease store is never stale as we use monitored colleciton Rid as id prefix for aux collection.
// Collection was removed and re-created, the rid would change.
// If it's not deleted, it's not stale. If it's deleted, it's not stale as it doesn't exist.
await this.leaseManager.CreateLeaseStoreIfNotExistsAsync();
string[] rangeIds = await this.EnumPartitionKeyRangeIds(this.collectionSelfLink);
Parallel.ForEach(rangeIds, async rangeId => {
this.statsSinceLastCheckpoint.AddOrUpdate(
rangeId,
new CheckpointStats(),
(partitionId, existingStats) => existingStats);
await this.leaseManager.CreateLeaseIfNotExistAsync(rangeId);
});
this.partitionManager = new PartitionManager <DocumentServiceLease>(this.HostName, this.leaseManager, this.options);
await this.partitionManager.SubscribeAsync(this);
await this.partitionManager.InitializeAsync();
}
async Task InitializeStoresAsync() //throws InterruptedException, ExecutionException, ExceptionWithAction
{
// Make sure the lease store exists
ILeaseManager leaseManager = this.host.LeaseManager;
if (!await leaseManager.LeaseStoreExistsAsync().ConfigureAwait(false))
{
await RetryAsync(() => leaseManager.CreateLeaseStoreIfNotExistsAsync(), null, "Failure creating lease store for this Event Hub, retrying",
"Out of retries creating lease store for this Event Hub", EventProcessorHostActionStrings.CreatingLeaseStore, 5).ConfigureAwait(false);
}
// else
// lease store already exists, no work needed
var partitionIds = await this.GetPartitionIdsAsync().ConfigureAwait(false);
// Now make sure the leases exist
var createLeaseTasks = new List <Task>();
foreach (string id in partitionIds)
{
var subjectId = id;
createLeaseTasks.Add(RetryAsync(() => leaseManager.CreateLeaseIfNotExistsAsync(subjectId), subjectId, $"Failure creating lease for partition {subjectId}, retrying",
$"Out of retries creating lease for partition {subjectId}", EventProcessorHostActionStrings.CreatingLease, 5));
}
await Task.WhenAll(createLeaseTasks).ConfigureAwait(false);
// Make sure the checkpoint store exists
ICheckpointManager checkpointManager = this.host.CheckpointManager;
if (!await checkpointManager.CheckpointStoreExistsAsync().ConfigureAwait(false))
{
await RetryAsync(() => checkpointManager.CreateCheckpointStoreIfNotExistsAsync(), null, "Failure creating checkpoint store for this Event Hub, retrying",
"Out of retries creating checkpoint store for this Event Hub", EventProcessorHostActionStrings.CreatingCheckpointStore, 5).ConfigureAwait(false);
}
// else
// checkpoint store already exists, no work needed
// Now make sure the checkpoints exist
var createCheckpointTasks = new List <Task>();
foreach (string id in partitionIds)
{
var subjectId = id;
createCheckpointTasks.Add(RetryAsync(() => checkpointManager.CreateCheckpointIfNotExistsAsync(subjectId), subjectId, $"Failure creating checkpoint for partition {subjectId}, retrying",
$"Out of retries creating checkpoint for partition {subjectId}", EventProcessorHostActionStrings.CreatingCheckpoint, 5));
}
await Task.WhenAll(createCheckpointTasks).ConfigureAwait(false);
}
private void ConfigureInstance(ILogger logger, IMetricFactory metricFactory, FixedProcessorClientOptions options, ILeaseManager leaseManager, ICheckpointManager checkpointManager)
{
Guard.NotNull(nameof(logger), logger);
Guard.NotNull(nameof(leaseManager), leaseManager);
Guard.NotNull(nameof(checkpointManager), checkpointManager);
Guard.NotNull(nameof(options), options);
Guard.NotNull(nameof(metricFactory), metricFactory);
_leaseManager = leaseManager;
_checkpointManager = checkpointManager;
_options = options;
_logger = logger;
_metricFactory = metricFactory;
_errorCounter = _metricFactory.CreateCounter("fpc-processing-error-count", "The number of processing errors raised since the start of the processor.", false, new string[0]);
_partitionStopProcessing = _metricFactory.CreateCounter("fpc-partition-stop-count", "The number of processing stop events raised since the start of the processor.", false, new string[0]);
_partitionInitializeProcessing = _metricFactory.CreateCounter("fpc-partition-init-count", "The number of processing initialization events raised since the start of the processor.", false, new string[0]);
}
public BufferedCheckpointManager(ICheckpointManager bufferCheckpointManager, ICheckpointManager targetCheckpointManager, int bufferInterval)
{
Guard.ArgumentNotNull(bufferCheckpointManager, nameof(bufferCheckpointManager));
Guard.ArgumentNotNull(targetCheckpointManager, nameof(targetCheckpointManager));
_bufferCheckpointManager = bufferCheckpointManager;
_targetCheckpointManager = targetCheckpointManager;
_bufferInterval = bufferInterval;
var bufferCheckpoint = _bufferCheckpointManager.GetCheckpoint();
var targetCheckpoint = _targetCheckpointManager.GetCheckpoint();
if (bufferCheckpoint > targetCheckpoint)
{
_targetCheckpointManager.SaveCheckpoint(bufferCheckpoint);
}
_timer = new PeriodicAction(SendBufferToTarget, bufferInterval);
}
private static PartitionContext MockPartitionContext(ICheckpointManager checkpointManager)
{
var ctor = typeof(PartitionContext).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, new[] { typeof(ICheckpointManager) }, null);
var partition = (PartitionContext)ctor?.Invoke(new object[] { checkpointManager });
Assert.NotNull(partition);
partition.Lease = new Lease()
{
Epoch = 1,
Offset = "10",
Owner = "EsSinkEventProcessorTest",
PartitionId = "0",
SequenceNumber = 17,
Token = Guid.NewGuid().ToString(),
};
return(partition);
}
/// <summary>
/// Creates a new <see cref="ConsumerManager"/> instance.
/// </summary>
/// <param name="connection">The event store connection to use for subscribing to streams</param>
/// <param name="checkpointManager">The checkpoint manager used for loading and saving consumer checkpoints</param>
/// <param name="consumers">The consumers to start</param>
/// <param name="logger">Application logger</param>
public ConsumerManager(
IEventStoreConnection connection,
ICheckpointManager checkpointManager,
IEnumerable <IConsumer> consumers,
Serilog.ILogger logger)
{
_connection = connection ?? throw new System.ArgumentNullException(nameof(connection));
_checkpointManager = checkpointManager ?? throw new System.ArgumentNullException(nameof(checkpointManager));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (consumers == null)
{
throw new ArgumentNullException(nameof(consumers));
}
_consumers = consumers
.Select(consumer => new RunningConsumer {
Consumer = consumer
}).ToList();
}
async Task InitializeStoresAsync() //throws InterruptedException, ExecutionException, ExceptionWithAction
{
// Make sure the lease store exists
ILeaseManager leaseManager = this.host.LeaseManager;
if (!await leaseManager.LeaseStoreExistsAsync().ConfigureAwait(false))
{
await RetryAsync(() => leaseManager.CreateLeaseStoreIfNotExistsAsync(), null, "Failure creating lease store for this Event Hub, retrying",
"Out of retries creating lease store for this Event Hub", EventProcessorHostActionStrings.CreatingLeaseStore, 5).ConfigureAwait(false);
}
// else
// lease store already exists, no work needed
// Now make sure the leases exist
foreach (string id in await this.GetPartitionIdsAsync().ConfigureAwait(false))
{
await RetryAsync(() => leaseManager.CreateLeaseIfNotExistsAsync(id), id, "Failure creating lease for partition, retrying",
"Out of retries creating lease for partition", EventProcessorHostActionStrings.CreatingLease, 5).ConfigureAwait(false);
}
// Make sure the checkpoint store exists
ICheckpointManager checkpointManager = this.host.CheckpointManager;
if (!await checkpointManager.CheckpointStoreExistsAsync().ConfigureAwait(false))
{
await RetryAsync(() => checkpointManager.CreateCheckpointStoreIfNotExistsAsync(), null, "Failure creating checkpoint store for this Event Hub, retrying",
"Out of retries creating checkpoint store for this Event Hub", EventProcessorHostActionStrings.CreatingCheckpointStore, 5).ConfigureAwait(false);
}
// else
// checkpoint store already exists, no work needed
// Now make sure the checkpoints exist
foreach (string id in await this.GetPartitionIdsAsync().ConfigureAwait(false))
{
await RetryAsync(() => checkpointManager.CreateCheckpointIfNotExistsAsync(id), id, "Failure creating checkpoint for partition, retrying",
"Out of retries creating checkpoint blob for partition", EventProcessorHostActionStrings.CreatingCheckpoint, 5).ConfigureAwait(false);
}
}
public void Initialize(Guid token, long _size, ICheckpointManager checkpointManager)
{
info.Initialize(token, _size);
checkpointManager.InitializeIndexCheckpoint(token);
main_ht_device = checkpointManager.GetIndexDevice(token);
}
public void Initialize(Guid token, int _version, ICheckpointManager checkpointManager)
{
info.Initialize(token, _version);
checkpointManager.InitializeLogCheckpoint(token);
}
private async ValueTask SimpleRecoveryTest1_Worker(CheckpointType checkpointType, ICheckpointManager checkpointManager, bool isAsync)
{
string checkpointDir = TestContext.CurrentContext.TestDirectory + $"/{TEST_CONTAINER}";
if (checkpointManager != null)
{
checkpointDir = null;
}
log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "/SimpleRecoveryTest1.log", deleteOnClose: true);
fht1 = new FasterKV
<AdId, NumClicks>
(128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = checkpointDir, CheckpointManager = checkpointManager, CheckPointType = checkpointType
}
);
fht2 = new FasterKV
<AdId, NumClicks>
(128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointDir = checkpointDir, CheckpointManager = checkpointManager, CheckPointType = checkpointType
}
);
int numOps = 5000;
var inputArray = new AdId[numOps];
for (int i = 0; i < numOps; i++)
{
inputArray[i].adId = i;
}
NumClicks value;
AdInput inputArg = default;
Output output = default;
var session1 = fht1.NewSession(new AdSimpleFunctions());
for (int key = 0; key < numOps; key++)
{
value.numClicks = key;
session1.Upsert(ref inputArray[key], ref value, Empty.Default, 0);
}
fht1.TakeFullCheckpoint(out Guid token);
fht1.CompleteCheckpointAsync().GetAwaiter().GetResult();
session1.Dispose();
if (isAsync)
{
await fht2.RecoverAsync(token);
}
else
{
fht2.Recover(token);
}
var session2 = fht2.NewSession(new AdSimpleFunctions());
for (int key = 0; key < numOps; key++)
{
var status = session2.Read(ref inputArray[key], ref inputArg, ref output, Empty.Default, 0);
if (status == Status.PENDING)
{
session2.CompletePending(true);
}
else
{
Assert.IsTrue(output.value.numClicks == key);
}
}
session2.Dispose();
log.Dispose();
fht1.Dispose();
fht2.Dispose();
if (checkpointManager == null)
{
new DirectoryInfo(checkpointDir).Delete(true);
}
}
public async ValueTask SimpleRecoveryTest2([Values] CheckpointType checkpointType, [Values] bool isAsync)
{
checkpointManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(TestUtils.MethodTestDir + "/checkpoints4"), false);
log = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/SimpleRecoveryTest2.log", deleteOnClose: true);
fht1 = new FasterKV <AdId, NumClicks>(128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointManager = checkpointManager, CheckPointType = checkpointType
}
);
fht2 = new FasterKV <AdId, NumClicks>(128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointManager = checkpointManager, CheckPointType = checkpointType
}
);
NumClicks value;
AdInput inputArg = default;
Output output = default;
var session1 = fht1.NewSession(new AdSimpleFunctions());
for (int key = 0; key < numOps; key++)
{
value.numClicks = key;
session1.Upsert(ref inputArray[key], ref value, Empty.Default, 0);
}
fht1.TakeFullCheckpoint(out Guid token);
fht1.CompleteCheckpointAsync().GetAwaiter().GetResult();
session1.Dispose();
if (isAsync)
{
await fht2.RecoverAsync(token);
}
else
{
fht2.Recover(token);
}
var session2 = fht2.NewSession(new AdSimpleFunctions());
for (int key = 0; key < numOps; key++)
{
var status = session2.Read(ref inputArray[key], ref inputArg, ref output, Empty.Default, 0);
if (status == Status.PENDING)
{
session2.CompletePending(true);
}
else
{
Assert.IsTrue(output.value.numClicks == key);
}
}
session2.Dispose();
}
public void Recover(Guid token, ICheckpointManager checkpointManager)
{
info.Recover(token, checkpointManager);
}
public async Task Start()
{
_isRunning = true;
ILeaseManager leaseManager = null;
ICheckpointManager checkpointManager = null;
switch (Settings.LeaseType)
{
case LeaseType.CloudSqlStorage:
SqlLeaseManager <PrototypeSqlLease> sqlLeaseManager =
new SqlLeaseManager <PrototypeSqlLease>(Name,
Settings.LeaseSqlConnectionString,
x => new PrototypeSqlLease(x),
x => new PrototypeSqlLease(x))
{
LeaseDuration = Settings.ExpireLeaseEvery,
};
leaseManager = sqlLeaseManager;
checkpointManager = sqlLeaseManager;
break;
case LeaseType.LocalDiskStorage:
DiskLeaseManager diskLeaseManager = new DiskLeaseManager(Settings.EventHubName, Settings.EventHubConsumerGroup, Name)
{
LeaseDuration = Settings.ExpireLeaseEvery,
};
leaseManager = diskLeaseManager;
checkpointManager = diskLeaseManager;
break;
}
if (leaseManager != null && checkpointManager != null)
{
// Custom construction that assigns a custom lease and checkpoint manager.
_eventProcessorHost = new EventProcessorHost(Name,
Settings.EventHubName,
Settings.EventHubConsumerGroup,
Settings.EventHubConnectionString,
checkpointManager,
leaseManager);
}
else
{
// The default construction uses Azure Storage lease and checkpoint management.
_eventProcessorHost = new EventProcessorHost(Name,
Settings.EventHubName,
Settings.EventHubConsumerGroup,
Settings.EventHubConnectionString,
Settings.LeaseConnectionString,
Settings.EventHubName);
}
_eventProcessorHost.PartitionManagerOptions = new PartitionManagerOptions
{
RenewInterval = Settings.RenewLeaseEvery,
LeaseDuration = Settings.ExpireLeaseEvery,
};
EventProcessorOptions options = new EventProcessorOptions
{
MaxBatchSize = Settings.ReadBatchSize,
PrefetchCount = Settings.ReadPrefetchCount,
ReceiveTimeout = Settings.ReadReceiveTimeout,
EnableReceiverRuntimeMetric = Settings.ReadEnableReceiverRuntimeMetric,
InvokeProcessorAfterReceiveTimeout = false,
};
options.SetExceptionHandler(UnhandledExceptionHandler);
await _eventProcessorHost.RegisterEventProcessorAsync <EventHubsReader>(options).ConfigureAwait(false);
}
public Task RegisterProcessorFactoryAsync(Lease lease, ICheckpointManager checkpointManager, IEventProcessorFactory eventProcessorFactory)
{
return(this.RegisterProcessorFactoryAsync(lease, checkpointManager, eventProcessorFactory, EventProcessorOptions.DefaultOptions));
}
