public void IsEventPositionEquivalentDetectsDifferentEnqueueTime()
{
var enqueueTime = DateTimeOffset.Parse("2015-10-27T12:00:00Z");
var trackOnePosition = TrackOne.EventPosition.FromEnqueuedTime(enqueueTime.UtcDateTime);
var trackTwoPosition = EventPosition.FromEnqueuedTime(enqueueTime.AddDays(1));
Assert.That(TrackOneComparer.IsEventPositionEquivalent(trackOnePosition, trackTwoPosition), Is.False);
}
public void IsEventPositionEquivalentRecognizesSameEnqueueTime()
{
var enqueueTime = DateTimeOffset.Parse("2015-10-27T12:00:00Z");
var trackOnePosition = TrackOne.EventPosition.FromEnqueuedTime(enqueueTime.UtcDateTime);
var trackTwoPosition = EventPosition.FromEnqueuedTime(enqueueTime);
Assert.That(TrackOneComparer.IsEventPositionEquivalent(trackOnePosition, trackTwoPosition), Is.True);
}
/// <inheritdoc/>
public async Task StartAsync()
{
await _lock.WaitAsync();
try {
if (_host != null)
{
_logger.Debug("Event processor host already running.");
return;
}
_logger.Debug("Starting event processor host...");
var consumerGroup = _hub.ConsumerGroup;
if (string.IsNullOrEmpty(consumerGroup))
{
consumerGroup = "$default";
}
_logger.Information($"Using Consumer Group: \"{consumerGroup}\"");
if (_lease != null && _checkpoint != null)
{
_host = new EventHubs.Processor.EventProcessorHost(
$"host-{Guid.NewGuid()}", _hub.EventHubPath, consumerGroup,
GetEventHubConnectionString(), _checkpoint, _lease);
}
else if (_config.BlobStorageConnString != null)
{
_host = new EventHubs.Processor.EventProcessorHost(
_hub.EventHubPath, consumerGroup, GetEventHubConnectionString(),
_config.BlobStorageConnString,
_config.LeaseContainerName ?? _hub.EventHubPath.ToSha1Hash());
}
else
{
_logger.Error("No checkpointing storage configured or checkpoint " +
"manager/lease manager implementation injected.");
throw new InvalidConfigurationException(
"Invalid checkpoint configuration.");
}
await _host.RegisterEventProcessorFactoryAsync(
_factory, new EventProcessorOptions {
InitialOffsetProvider = s =>
EventPosition.FromEnqueuedTime(DateTime.UtcNow),
MaxBatchSize = _config.ReceiveBatchSize,
ReceiveTimeout = _config.ReceiveTimeout,
InvokeProcessorAfterReceiveTimeout = true
});
_logger.Information("Event processor host started.");
}
catch (Exception ex) {
_logger.Error(ex, "Error starting event processor host.");
_host = null;
throw ex;
}
finally {
_lock.Release();
}
}
protected async Task VerifyDataOnIoTHubAsync()
{
// Leaf device without parent not expected to send messages
if (!this.edgeDeviceId.HasValue)
{
return;
}
var builder = new EventHubsConnectionStringBuilder(this.eventhubCompatibleEndpointWithEntityPath)
{
TransportType = this.eventHubClientTransportType
};
Console.WriteLine($"Receiving events from device '{this.context.Device.Id}' on Event Hub '{builder.EntityPath}'");
EventHubClient eventHubClient =
EventHubClient.CreateFromConnectionString(builder.ToString());
this.proxy.ForEach(p => eventHubClient.WebProxy = p);
PartitionReceiver eventHubReceiver = eventHubClient.CreateReceiver(
"$Default",
EventHubPartitionKeyResolver.ResolveToPartition(
this.context.Device.Id,
(await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
EventPosition.FromEnqueuedTime(DateTime.Now.AddMinutes(-5)));
var result = new TaskCompletionSource <bool>();
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(3)))
{
using (cts.Token.Register(() => result.TrySetCanceled()))
{
eventHubReceiver.SetReceiveHandler(
new PartitionReceiveHandler(
eventData =>
{
eventData.SystemProperties.TryGetValue("iothub-connection-device-id", out var devId);
if (devId != null && devId.ToString().Equals(this.context.Device.Id, StringComparison.Ordinal) &&
Encoding.UTF8.GetString(eventData.Body).Contains(this.context.MessageGuid, StringComparison.Ordinal))
{
result.TrySetResult(true);
return(true);
}
return(false);
}));
await result.Task;
}
}
await eventHubReceiver.CloseAsync();
await eventHubClient.CloseAsync();
}
private PartitionReceiver CreateReceiver(ILogger log, Configuration configuration, string partition, string offset)
{
var epoch = DateTime.UtcNow.Ticks;
if (string.IsNullOrWhiteSpace(offset))
{
if (DefaultAge.HasValue)
{
var timestamp = DateTime.UtcNow.Subtract(DefaultAge.Value);
var position = EventPosition.FromEnqueuedTime(timestamp);
if (UseEpochReceiver)
{
log.Debug("Creating epoch {Epoch} receiver for {Consumer}#{Partition} from time {Timestamp}",
epoch, configuration.ConsumerGroup, partition, timestamp);
return(configuration.Client.CreateEpochReceiver(configuration.ConsumerGroup, partition, position, epoch));
}
else
{
log.Debug("Creating receiver for {Consumer}#{Partition} from time {Timestamp}",
configuration.ConsumerGroup, partition, timestamp);
return(configuration.Client.CreateReceiver(configuration.ConsumerGroup, partition, position));
}
}
else
{
var position = DefaultPosition;
if (UseEpochReceiver)
{
log.Debug("Creating epoch {Epoch} receiver for {Consumer}#{Partition} from position {Position}",
epoch, configuration.ConsumerGroup, partition, position);
return(configuration.Client.CreateEpochReceiver(configuration.ConsumerGroup, partition, position, epoch));
}
else
{
log.Debug("Creating receiver for {Consumer}#{Partition} from offset {Position}",
configuration.ConsumerGroup, partition, position);
return(configuration.Client.CreateReceiver(configuration.ConsumerGroup, partition, position));
}
}
}
else
{
var position = EventPosition.FromOffset(offset);
if (UseEpochReceiver)
{
log.Debug("Creating epoch {Epoch} receiver for {Consumer}#{Partition} from saved offset {Offset}",
epoch, configuration.ConsumerGroup, partition, offset);
return(configuration.Client.CreateEpochReceiver(configuration.ConsumerGroup, partition, position, epoch));
}
else
{
log.Debug("Creating receiver for {Consumer}#{Partition} from saved offset {Offset}",
configuration.ConsumerGroup, partition, offset);
return(configuration.Client.CreateReceiver(configuration.ConsumerGroup, partition, position));
}
}
}
/// <summary>
/// Determines the default start position for processing when no checkpoint is found
/// </summary>
/// <returns>An event position indicating the startup logic</returns>
public static EventPosition GetStartPosition()
{
EventPosition startPosition;
var startDefinition = Environment.GetEnvironmentVariable("StartPosition");
if (!bool.TryParse(Environment.GetEnvironmentVariable("StartInclusive"), out var startInclusive))
{
startInclusive = false;
}
if (string.IsNullOrWhiteSpace(startDefinition))
{
startDefinition = "Start";
}
startDefinition = startDefinition.Trim().ToLowerInvariant();
if (string.IsNullOrWhiteSpace(startDefinition))
{
startDefinition = "Start";
}
switch (startDefinition)
{
case "end":
startPosition = EventPosition.FromEnd();
break;
case "offset":
startPosition = EventPosition.FromOffset(Environment.GetEnvironmentVariable("StartOffset"), startInclusive);
break;
case "sequence":
if (!long.TryParse(Environment.GetEnvironmentVariable("StartSequence"), out var startSequence))
{
startSequence = -1;
}
startPosition = EventPosition.FromSequenceNumber(startSequence, startInclusive);
break;
case "time":
if (!int.TryParse(Environment.GetEnvironmentVariable("StartSeconds"), out var startSeconds))
{
startSeconds = 0;
}
startPosition = EventPosition.FromEnqueuedTime(DateTime.UtcNow.AddSeconds(startSeconds * -1));
break;
default:
startPosition = EventPosition.FromStart();
break;
}
return(startPosition);
}
public void TheSameEnqueueTimesAreEqual()
{
var first = EventPosition.FromEnqueuedTime(DateTimeOffset.Parse("2015-10-27T00:00:00Z"));
var second = EventPosition.FromEnqueuedTime(DateTimeOffset.Parse("2015-10-27T00:00:00Z"));
Assert.That(first.Equals((object)second), Is.True, "The default Equals comparison is incorrect.");
Assert.That(first.Equals(second), Is.True, "The IEquatable comparison is incorrect.");
Assert.That((first == second), Is.True, "The == operator comparison is incorrect.");
Assert.That((first != second), Is.False, "The != operator comparison is incorrect.");
}
public void BuildFilterExpressionIgnoresInclusiveFlagForEnqueuedTime(bool inclusive)
{
var position = EventPosition.FromEnqueuedTime(DateTimeOffset.Parse("2015-10-27T12:00:00Z"));
position.IsInclusive = inclusive;
var filter = AmqpFilter.BuildFilterExpression(position);
Assert.That(filter, Does.Not.Contain("="), "The comparison should not consider the inclusive flag.");
}
public async Task ReadPartitionFromDate()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionFromDate
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
DateTimeOffset oneHourAgo = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(1));
EventPosition startingPosition = EventPosition.FromEnqueuedTime(oneHourAgo);
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
cancellationSource.Token))
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public void BuildFilterExpressionUsesEnqueuedTime()
{
// Set all properties for the event position.
var enqueuedTime = DateTimeOffset.Parse("2015-10-27T12:00:00Z");
var position = EventPosition.FromEnqueuedTime(enqueuedTime);
var filter = AmqpFilter.BuildFilterExpression(position);
Assert.That(filter, Contains.Substring(AmqpFilter.EnqueuedTimeName), "The enqueued time should have been used.");
Assert.That(filter, Contains.Substring(enqueuedTime.ToUnixTimeMilliseconds().ToString()), "The enqueued time value should be present in the filter.");
}
public async Task <bool> RegisterEventMessageConsumerAsync <TEventProcessor>(string name)
where TEventProcessor : EventProcessorDefault
{
ConsumerConfigurationsOptions consumerConfiguration = GetConsumerConfiguration(name);
if (EventProcessorHostList != null && EventProcessorHostList.Any(s => s.EventHubPath == consumerConfiguration.EventHubName))
{
return(false);
}
eventProcessorHost = new EventProcessorHost(
consumerConfiguration.EventHubName,
!string.IsNullOrWhiteSpace(consumerConfiguration.ConsumerGroupName) ? consumerConfiguration.ConsumerGroupName : PartitionReceiver.DefaultConsumerGroupName,
consumerConfiguration.ConnectionString,
consumerConfiguration.StorageConnectionString,
consumerConfiguration.StorageContainerName)
{
PartitionManagerOptions = new PartitionManagerOptions
{
RenewInterval = TimeSpan.FromSeconds(consumerConfiguration.RenewIntervalInSeconds),
LeaseDuration = TimeSpan.FromSeconds(consumerConfiguration.LeaseDurationInSeconds)
}
};
var eventProcessorOptions = new EventProcessorOptions()
{
MaxBatchSize = consumerConfiguration.NumberOfEventsPerRequest,
ReceiveTimeout = TimeSpan.FromSeconds(consumerConfiguration.ReceiveTimeoutInSeconds)
};
DateTime offsetStartDateTime;
if (!string.IsNullOrEmpty(consumerConfiguration.OffsetStartDateTime) && DateTime.TryParse(consumerConfiguration.OffsetStartDateTime, out offsetStartDateTime))
{
eventProcessorOptions.InitialOffsetProvider = (partitionId) => EventPosition.FromEnqueuedTime(offsetStartDateTime);
}
else
{
eventProcessorOptions.InitialOffsetProvider = (partitionId) => EventPosition.FromStart();
}
EventProcessorFactory <TEventProcessor> eventProcessorFactory = new EventProcessorFactory <TEventProcessor>(consumerConfiguration, ServiceProvider, Configuration);
await eventProcessorHost.RegisterEventProcessorFactoryAsync(eventProcessorFactory, eventProcessorOptions);
EventProcessorHostList.Add(eventProcessorHost);
EventProcessorFactoryList.Add(name, eventProcessorFactory);
return(true);
}
/// <summary>
/// Returns the event process options
/// </summary>
private EventProcessorOptions GetEventProcessorOptions(int offsetInSeconds)
{
var options = new EventProcessorOptions()
{
MaxBatchSize = 500,
PrefetchCount = 500,
ReceiveTimeout = TimeSpan.FromSeconds(20),
InitialOffsetProvider = (partitionId) => EventPosition.FromEnqueuedTime(DateTime.UtcNow.AddSeconds(-offsetInSeconds)),
};
options.SetExceptionHandler(new Action <ExceptionReceivedEventArgs>(EventHub_ExceptionReceived));
return(options);
}
/// <summary>
/// Provides test cases for the equality tests.
/// </summary>
///
public static IEnumerable <object[]> IsEquivalentToDetectsEqualEventPositionsCases()
{
var date = DateTimeOffset.Parse("1975-04-04T00:00:00Z");
yield return(new object[] { EventPosition.Earliest, EventPosition.Earliest });
yield return(new object[] { EventPosition.Latest, EventPosition.Latest });
yield return(new object[] { EventPosition.FromOffset(1975), EventPosition.FromOffset(1975) });
yield return(new object[] { EventPosition.FromSequenceNumber(42), EventPosition.FromSequenceNumber(42) });
yield return(new object[] { EventPosition.FromEnqueuedTime(date), EventPosition.FromEnqueuedTime(date) });
}
public async Task StartAsync(CancellationToken cancellationToken)
{
eventProcessorHost = new EventProcessorHost(
"plantgrow",
"realtimeweb",
"Azure-IoT-hub-endpoint-url"
, "Azure-Storage_account-Access_keys-connection_string-url", "data1");
// Registers the Event Processor Host and starts receiving messages
await eventProcessorHost.RegisterEventProcessorFactoryAsync(new AzureStreamProcessorFactory(_subService), new EventProcessorOptions
{
InitialOffsetProvider = (pid) => EventPosition.FromEnqueuedTime(DateTime.UtcNow)
});
}
// Asynchronously create a PartitionReceiver for a partition and then start
// reading any messages sent from the simulated client.
private static async Task ReceiveMessagesFromDeviceAsync(string partition, CancellationToken ct)
{
// Create the receiver using the default consumer group.
// For the purposes of this sample, read only messages sent since
// the time the receiver is created. Typically, you don't want to skip any messages.
var eventHubReceiver =
_eventHubClient.CreateReceiver("$Default", partition, EventPosition.FromEnqueuedTime(DateTime.Now));
_logger.LogInformation("Create receiver on partition: " + partition);
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
_logger.LogInformation("Listening for messages on: " + partition);
// Check for EventData - this methods times out if there is nothing to retrieve.
var events = await eventHubReceiver.ReceiveAsync(100);
// If there is data in the batch, process it.
if (events == null)
{
continue;
}
foreach (var eventData in events)
{
var data = Encoding.UTF8.GetString(eventData.Body.Array);
_logger.LogInformation($"Message received on partition {partition}:");
_logger.LogInformation($"Data: {data}:");
_logger.LogInformation("Application properties (set by device):");
var sb = new StringBuilder();
foreach (var(key, value) in eventData.Properties)
{
sb.Append($"{key}: {value} \r\n");
_logger.LogInformation(sb.ToString());
}
sb.Clear();
_logger.LogInformation("System properties (set by IoT Hub):");
foreach (var(key, value) in eventData.SystemProperties)
{
sb.Append($"{key}: {value},");
}
_logger.LogInformation(sb.ToString());
}
}
}
public void ToStringReflectsTheState()
{
var inclusive = true;
var offset = 123;
var sequence = 778;
var enqueued = DateTimeOffset.Now.AddHours(1);
Assert.That(EventPosition.Earliest.ToString(), Contains.Substring(nameof(EventPosition.Earliest)), "Earliest should be represented.");
Assert.That(EventPosition.Latest.ToString(), Contains.Substring(nameof(EventPosition.Latest)), "Latest should be represented.");
Assert.That(EventPosition.FromOffset(offset).ToString(), Contains.Substring($"[{ offset }]"), "The offset should be represented.");
Assert.That(EventPosition.FromSequenceNumber(sequence).ToString(), Contains.Substring($"[{ sequence }]"), "The sequence should be represented.");
Assert.That(EventPosition.FromEnqueuedTime(enqueued).ToString(), Contains.Substring($"[{ enqueued }]"), "The enqueued time should be represented.");
Assert.That(EventPosition.FromOffset(offset, inclusive).ToString(), Contains.Substring($"[{ inclusive }]"), "The inclusive flag should be represented for the offset.");
Assert.That(EventPosition.FromSequenceNumber(sequence, inclusive).ToString(), Contains.Substring($"[{ inclusive }]"), "The inclusive flag should be represented for the sequence number.");
}
static async Task ReceiveMessagesAsync(DateTime lastReceivedMesssage)
{
var builder = new EventHubsConnectionStringBuilder(Settings.Current.EventHubConnectionString);
Logger.LogInformation($"Receiving events from device '{Settings.Current.DeviceId}' on Event Hub '{builder.EntityPath}' with last received message at {lastReceivedMesssage.ToString()}");
EventHubClient eventHubClient =
EventHubClient.CreateFromConnectionString(builder.ToString());
PartitionReceiver eventHubReceiver = eventHubClient.CreateReceiver(
Settings.Current.ConsumerGroupId,
EventHubPartitionKeyResolver.ResolveToPartition(Settings.Current.DeviceId, (await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
EventPosition.FromEnqueuedTime(lastReceivedMesssage == DateTime.MinValue ? DateTime.UtcNow : lastReceivedMesssage));
eventHubReceiver.SetReceiveHandler(new PartitionReceiveHandler(Settings.Current.DeviceId, Settings.Current.ExcludedModuleIds));
}
static async Task ReceiveMessages()
{
var builder = new EventHubsConnectionStringBuilder(Settings.Current.EventHubConnectionString);
Log.LogInformation($"Receiving events from device '{Settings.Current.DeviceId}' on Event Hub '{builder.EntityPath}'");
EventHubClient eventHubClient =
EventHubClient.CreateFromConnectionString(builder.ToString());
PartitionReceiver eventHubReceiver = eventHubClient.CreateReceiver(
"$Default",
EventHubPartitionKeyResolver.ResolveToPartition(Settings.Current.DeviceId, (await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
EventPosition.FromEnqueuedTime(DateTime.UtcNow));
eventHubReceiver.SetReceiveHandler(new PartitionReceiveHandler(Settings.Current.DeviceId, Settings.Current.ExcludedModuleIds));
}
private static async Task ReceiveMessagesFromDeviceAsync(string partitionId, CancellationToken cts)
{
// Create the receiver using the default consumer group.
var eventHubReceiver = _eventHubClient
.CreateReceiver("$Default", partitionId, EventPosition.FromEnqueuedTime(DateTime.UtcNow));
Console.WriteLine("Create receiver on partition: " + partitionId);
while (true)
{
if (cts.IsCancellationRequested)
{
break;
}
Console.WriteLine($"Listening for messages on: {partitionId}");
// Check for EventData - this methods times out if there is nothing to retrieve.
var events = await eventHubReceiver.ReceiveAsync(100);
// If there is data in the batch, process it.
if (events == null)
{
continue;
}
foreach (EventData eventData in events)
{
string data = Encoding.UTF8.GetString(eventData.Body.Array);
Console.WriteLine($"Message received on partition {partitionId}:");
Console.WriteLine("=======================================");
Console.WriteLine($" Message: {data}");
Console.WriteLine("=======================================");
Console.WriteLine("Application properties (set by device):");
foreach (var prop in eventData.Properties)
{
Console.WriteLine($" {prop.Key}: {prop.Value}");
}
Console.WriteLine("System properties (set by IoT Hub):");
foreach (var prop in eventData.SystemProperties)
{
Console.WriteLine($" {prop.Key}: {prop.Value}");
}
}
}
}
private async Task SetupEphAsync()
{
currentEPH = new EventProcessorHost(
eventHubName,
PartitionReceiver.DefaultConsumerGroupName,
geoDrAliasConnectionString,
storageConnectionString,
storageContainers[currentContainerIdx]);
var options = new EventProcessorOptions();
if (storageContainers.Length == 1)
{
options.InitialOffsetProvider = (p) => EventPosition.FromEnqueuedTime(DateTime.UtcNow);
}
await currentEPH.RegisterEventProcessorFactoryAsync(new GeoDrEventConsumerFactory(storageContainers.Length > 1, this), options);
}
// Asynchronously create a PartitionReceiver for a partition and then start
// reading any messages sent from the simulated client.
private static async Task ReceiveMessagesFromDeviceAsync(string partition, CancellationToken ct)
{
// Create the receiver using the default consumer group.
// For the purposes of this sample, read only messages sent since
// the time the receiver is created. Typically, you don't want to skip any messages.
var eventHubReceiver =
s_eventHubClient.CreateReceiver("$Default", partition, EventPosition.FromEnqueuedTime(DateTime.Now));
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
Console.WriteLine("Listening for messages on: " + partition);
// Check for EventData - this methods times out if there is nothing to retrieve.
var events = await eventHubReceiver.ReceiveAsync(100);
// If there is data in the batch, process it.
if (events == null)
{
continue;
}
foreach (EventData eventData in events)
{
Console.WriteLine("-------- Message Recieved --------");
string data = Encoding.UTF8.GetString(eventData.Body.Array);
Console.WriteLine("Message received on partition {0}:", partition);
Console.WriteLine(" {0}:", data);
Console.WriteLine("Application properties (set by device):");
foreach (var prop in eventData.Properties)
{
Console.WriteLine(" {0}: {1}", prop.Key, prop.Value);
}
Console.WriteLine("System properties (set by IoT Hub):");
foreach (var prop in eventData.SystemProperties)
{
Console.WriteLine(" {0}: {1}", prop.Key, prop.Value);
}
Console.WriteLine("----------------------------------");
}
}
}
public void CreateConsumerCreatesDefaultWhenNoOptionsArePassed()
{
var expectedOptions = new EventHubConsumerClientOptions();
var expectedConsumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var expectedPartition = "56767";
var expectedPosition = EventPosition.FromEnqueuedTime(DateTimeOffset.Parse("2015-10-27T12:00:00Z"));
var connectionString = "Endpoint=value.com;SharedAccessKeyName=[value];SharedAccessKey=[value];EntityPath=[value]";
var mockClient = new ReadableOptionsMock(connectionString, new EventHubConnectionOptions());
mockClient.CreateTransportConsumer(expectedConsumerGroup, expectedPartition, expectedPosition);
EventHubConsumerClientOptions actualOptions = mockClient.ConsumerOptions;
Assert.That(actualOptions, Is.Not.Null, "The consumer options should have been set.");
Assert.That(actualOptions.OwnerLevel, Is.EqualTo(expectedOptions.OwnerLevel), "The owner levels should match.");
Assert.That(actualOptions.PrefetchCount, Is.EqualTo(expectedOptions.PrefetchCount), "The prefetch counts should match.");
Assert.That(actualOptions.RetryOptions.IsEquivalentTo(expectedOptions.RetryOptions), Is.True, "The retries should match.");
Assert.That(actualOptions.MaximumReceiveWaitTimeOrDefault, Is.EqualTo(expectedOptions.MaximumReceiveWaitTimeOrDefault), "The wait times should match.");
}
// Asynchronously create a PartitionReceiver for a partition and then start
// reading any messages sent from the simulated client.
private static async Task ReceiveMessagesFromDeviceAsync(EventHubClient eventHub, string partition, CancellationToken ct)
{
try
{
// Create the receiver using the default consumer group.
// For the purposes of this sample, read only messages sent since
// the time the receiver is created. Typically, you don't want to skip any messages.
var eventHubReceiver =
eventHub.CreateReceiver("$Default", partition, EventPosition.FromEnqueuedTime(DateTime.Now));
Sitecore.Diagnostics.Log.Info("Create receiver on partition: " + partition, eventHub);
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
Sitecore.Diagnostics.Log.Info("Listening for messages on: " + partition, eventHub);
// Check for EventData - this methods times out if there is nothing to retrieve.
var events = await eventHubReceiver.ReceiveAsync(100);
if (events == null)
{
continue;
}
// If there is data in the batch, process it.
foreach (var eventData in events)
{
var args = new MessageReceivedArgs
{
EventData = eventData, EventHubClient = eventHub,
Partition = partition, Database = _masterDb
};
CorePipeline.Run("iotHub.MessageReceived", args);
}
}
}
catch (Exception e)
{
Sitecore.Diagnostics.Log.Error($"Error processing Device to Cloud events", e, e);
}
}
protected override async Task ExecuteAsync(CancellationToken cancellationToken)
{
this.logger.LogInformation("Test Result Event Receiving Service running.");
DateTime eventEnqueuedFrom = DateTime.UtcNow;
var builder = new EventHubsConnectionStringBuilder(Settings.Current.EventHubConnectionString);
this.logger.LogDebug($"Receiving events from device '{Settings.Current.DeviceId}' on Event Hub '{builder.EntityPath}' enqueued on or after {eventEnqueuedFrom}");
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(builder.ToString());
PartitionReceiver eventHubReceiver = eventHubClient.CreateReceiver(
Settings.Current.ConsumerGroupName,
EventHubPartitionKeyResolver.ResolveToPartition(Settings.Current.DeviceId, (await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
EventPosition.FromEnqueuedTime(eventEnqueuedFrom));
eventHubReceiver.SetReceiveHandler(new PartitionReceiveHandler(Settings.Current.TrackingId, Settings.Current.DeviceId, this.storage));
await cancellationToken.WhenCanceled();
this.logger.LogInformation($"Finish ExecuteAsync method in {nameof(TestResultEventReceivingService)}");
}
private static async Task ReceiveMessagesFromDeviceAsync(string partition, CancellationToken ct)
{
var eventHubReceiver =
_eventHubClient.CreateReceiver("$Default", partition, EventPosition.FromEnqueuedTime(DateTime.Now));
// Console.WriteLine("Create receiver on partition: " + partition);
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
// Console.WriteLine("Listening for messages on: " + partition);
var events = await eventHubReceiver.ReceiveAsync(30, TimeSpan.FromSeconds(15));
if (events == null)
{
continue;
}
foreach (var eventData in events)
{
var data = Encoding.UTF8.GetString(eventData.Body.Array);
Console.WriteLine(">>> Message received on partition {0}:", partition);
Console.WriteLine(" {0}:", data);
Console.WriteLine("Application properties (set by device):");
foreach (var prop in eventData.Properties)
{
Console.WriteLine(" {0}: {1}", prop.Key, prop.Value);
}
Console.WriteLine("System properties (set by IoT Hub):");
foreach (var prop in eventData.SystemProperties)
{
Console.WriteLine(" {0}: {1}", prop.Key, prop.Value);
}
}
}
}
public async void StartListening(GapDetection detectionEvent)
{
if (gapDetectionEvent != null)
{
return;
}
try
{
var connectionString =
new EventHubsConnectionStringBuilder(
new Uri(GapConfig.EventHubsCompatibleEndpoint),
GapConfig.EventHubsCompatiblePath,
GapConfig.IotHubSasKeyName,
GapConfig.IotHubSasKey);
eventHubClient = EventHubClient.CreateFromConnectionString(connectionString.ToString());
// Create a PartitionReciever for each partition on the hub.
var runtimeInfo = await eventHubClient.GetRuntimeInformationAsync();
var d2cPartitions = runtimeInfo.PartitionIds;
// TODO: listening on partition 0 only.
receivers =
d2cPartitions
.Select(d2c => eventHubClient.CreateReceiver("$Default", d2c, EventPosition.FromEnqueuedTime(DateTime.Now)))
.ToList();
gapTimer = ThreadPoolTimer.CreatePeriodicTimer(BackgroundListen, timerPeriod);
GapDetectionEvent += detectionEvent;
gapDetectionEvent = detectionEvent;
}
catch (Exception e)
{
Debug.WriteLine(e.ToString());
}
}
private Task InitializeEventHandler(PartitionInitializingEventArgs args)
{
try
{
if (args.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
log.LogDebug($"Initialize EventHub partition: { args.PartitionId } from EnqueueTime {utcMonitorStart}");
// If no checkpoint was found, start processing events enqueued now -5 minutes or in the future.
// This should be the case -- rely on timestamp, not a checkpoint to ensure nothing is missed across multiple instances
EventPosition startPositionWhenNoCheckpoint = EventPosition.FromEnqueuedTime(utcMonitorStart);
args.DefaultStartingPosition = startPositionWhenNoCheckpoint;
}
catch
{
}
return(Task.CompletedTask);
}
private void ConfigureEventProcessorHostServices(IServiceCollection services)
{
services.AddSingleton <IEventProcessor, JobEventProcessor>();
services.AddSingleton <IEventProcessorFactory, IotHubEventProcessorFactory>();
services.AddSingleton <IEventProcessorHostConfig>(new EventProcessorHostConfig()
{
EventHubConnectionString = Configuration.GetValue <string>("OrderManagerEventHubConnectionString"),
ConsumerGroupName = Configuration.GetValue <string>("OrderManagerEventHubConsumerGroup"),
EventHubPath = Configuration.GetValue <string>("OrderManagerEventHubPath"),
StorageConnectionString = Configuration.GetValue <string>("BlobStorageConnectionString"),
LeaseContainerName = "orderleases"
});
services.Configure <EventProcessorOptions>(options =>
{
options.MaxBatchSize = 10;
options.PrefetchCount = 100;
options.InitialOffsetProvider = (partitionId) => EventPosition.FromEnqueuedTime(DateTime.UtcNow);
});
services.AddHostedService <IotHubListener>();
}
private static async Task ReceiveFromSinglePartition(string connectionString)
{
Console.WriteLine("Connecting to the Event Hub...");
var eventHubClient = EventHubClient.CreateFromConnectionString(connectionString);
//var partitionReceiver = eventHubClient.CreateReceiver("$Default", "0", EventPosition.FromStart());
var partitionReceiver = eventHubClient.CreateReceiver("$Default", "0", EventPosition.FromEnqueuedTime(DateTime.Now));
Console.WriteLine("Waiting for incoming events ....");
while (true)
{
var messages = await partitionReceiver.ReceiveAsync(5).ConfigureAwait(false);
if (messages != null)
{
foreach (var eventData in messages)
{
var receivedMessage = Encoding.UTF8.GetString(eventData.Body.Array);
Console.WriteLine($"{receivedMessage} : PartitionId: {partitionReceiver.PartitionId}");
}
}
}
}
private static async Task MainAsync(string[] args)
{
AppDomain.CurrentDomain.ProcessExit += OnExit;
AssemblyLoadContext.Default.Unloading += OnExit;
// Setup the Options
var epo = new EventProcessorOptions
{
InitialOffsetProvider = (partitionId) => EventPosition.FromEnqueuedTime(DateTime.UtcNow)
};
// Setup the configuration File
var config = new Configuration();
eventProcessorHost = new EventProcessorHost(
config.Hub,
PartitionReceiver.DefaultConsumerGroupName,
config.EventHubEndpoint,
config.StorageConnectionString,
config.StorageContainer);
await eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(epo);
}