/// <summary>
/// A <i>potential</i> poisoned <see cref="EventHubs.EventData"/> has been identified and needs to be orchestrated.
/// </summary>
/// <param name="event">The corresponding <see cref="EventHubs.EventData"/>.</param>
/// <param name="exception">The corresponding <see cref="Exception"/>.</param>
/// <remarks>A <see cref="PoisonMessage"/> will be written to Azure table storage.</remarks>
public async Task SetAsync(EventHubs.EventData @event, Exception exception)
{
if (@event == null)
{
throw new ArgumentNullException(nameof(@event));
}
if (exception == null)
{
throw new ArgumentNullException(nameof(exception));
}
var msg = new PoisonMessage(_storagePartitionKey, _storageRowKey)
{
Offset = @event.SystemProperties.Offset,
SequenceNumber = @event.SystemProperties.SequenceNumber,
EnqueuedTimeUtc = @event.SystemProperties.EnqueuedTimeUtc,
PoisonedTimeUtc = DateTime.UtcNow,
FunctionType = _args.Options !.FunctionType,
FunctionName = _args.Options !.FunctionName,
Body = Substring(Encoding.UTF8.GetString(@event.Body.Array)),
Exception = Substring(exception.ToString())
};
await _poisonTable.ExecuteAsync(TableOperation.InsertOrReplace(msg)).ConfigureAwait(false);
}
/// <summary>
/// Checks whether the <see cref="EventHubs.EventData"/> is in a <b>Poison</b> state and determines the corresponding <see cref="PoisonMessageAction"/>.
/// </summary>
/// <param name="event">The corresponding <see cref="EventData"/>.</param>
/// <returns>The resulting <see cref="PoisonMessageAction"/>.</returns>
/// <remarks>Reads the <see cref="PoisonMessage"/> in Azure table storage to determine the result. Where <see cref="PoisonMessage.SkipMessage"/> is <c>true</c>, then
/// <see cref="PoisonMessageAction.PoisonSkip"/> will be returned; otherwise, <see cref="PoisonMessageAction.PoisonRetry"/>.</remarks>
public async Task <PoisonMessageAction> CheckAsync(EventHubs.EventData @event)
{
if (@event == null)
{
throw new ArgumentNullException(nameof(@event));
}
var msg = await GetPoisonMessageAsync().ConfigureAwait(false);
if (msg == null)
{
return(PoisonMessageAction.NotPoison);
}
if (@event.SystemProperties.SequenceNumber != msg.SequenceNumber)
{
// Warn if event exists with different offset - this means things are slightly out of whack!
_args.Logger.LogWarning($"EventData (Seq#: '{@event.SystemProperties.SequenceNumber}') being processed is out of sync with persisted Poison Message (Seq#: '{msg.SequenceNumber}'); EventData assumed correct and Poison Message deleted.");
await RemoveAsync(@event, PoisonMessageAction.Undetermined).ConfigureAwait(false);
}
if (msg.SkipMessage)
{
return(PoisonMessageAction.PoisonSkip);
}
else
{
return(PoisonMessageAction.PoisonRetry);
}
}
public static EventHubs.EventData CreateEvent(int exceptionCount)
{
var e = new EventHubs.EventData(Encoding.UTF8.GetBytes(Entities.Cleaner.Clean(DateTime.Now).ToString()));
e.Properties.Add("ExceptionCount", exceptionCount);
return(e);
}
public static void Run([CosmosDBTrigger(
databaseName: "IoT",
collectionName: "IoT1",
ConnectionStringSetting = "DBConnection",
LeaseCollectionName = "leases")] IReadOnlyList <Document> input, TraceWriter log)
{
string connectionString = ConfigurationSettings.AppSettings["EventHubConnection"];
var connectionStringBuilder = new EventHubsConnectionStringBuilder(connectionString)
{
EntityPath = "rafat-eventhub1"
};
var client = Microsoft.Azure.EventHubs.EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
foreach (var doc in input)
{
string json = string.Format("{{\"iotid\":\"{0}\",\"temp\":{1}}}", doc.GetPropertyValue <string>("iotid"), doc.GetPropertyValue <string>("temp"));
EventData data = new Microsoft.Azure.EventHubs.EventData(Encoding.UTF8.GetBytes(json));
client.SendAsync(data);
}
if (input != null && input.Count > 0)
{
log.Verbose("Documents modified " + input.Count);
log.Verbose("First document Id " + input[0].Id);
}
}
/// <summary>
/// Publishes the <paramref name="events"/>.
/// </summary>
/// <param name="events">The <see cref="EventData"/> instances.</param>
/// <returns>The corresponding <see cref="Task"/>.</returns>
protected override async Task PublishEventsAsync(params EventData[] events)
{
if (events == null || events.Length == 0)
{
return;
}
var partitionKey = GetPartitionKey(events);
var eventHubEvents = new EventHubs.EventData[events.Length];
for (int i = 0; i < events.Length; i++)
{
eventHubEvents[i] = events[i].ToEventHubsEventData();
}
try
{
await _invoker.InvokeAsync(this, async() => await _client.SendAsync(eventHubEvents, partitionKey).ConfigureAwait(false)).ConfigureAwait(false);
}
#pragma warning disable CA1031 // Do not catch general exception types; by-design, is a catch all.
catch (Exception ex)
#pragma warning restore CA1031
{
OnException(events, partitionKey, ex);
if (!SwallowException)
{
throw;
}
}
}
/// <summary>
/// A previously identified poisoned <see cref="EventHubs.EventData"/> has either successfully processed or can be skipped and should be removed.
/// </summary>
/// <param name="event">The corresponding <see cref="EventHubs.EventData"/>.</param>
/// <param name="action">The corresponding reason (<see cref="PoisonMessageAction"/>) for removal.</param>
/// <remarks>The corresponding <see cref="PoisonMessage"/> will be removed/deleted from Azure table storage.</remarks>
public async Task RemoveAsync(EventHubs.EventData @event, PoisonMessageAction action)
{
if (@event == null)
{
throw new ArgumentNullException(nameof(@event));
}
var msg = await GetPoisonMessageAsync().ConfigureAwait(false);
if (msg == null)
{
return;
}
// Audit the skipped record.
if (action == PoisonMessageAction.PoisonSkip)
{
msg.SkippedTimeUtc = DateTime.UtcNow;
msg.RowKey = msg.SkippedTimeUtc.Value.ToString("o", System.Globalization.CultureInfo.InvariantCulture) + "-" + msg.RowKey;
await _skippedTable.ExecuteAsync(TableOperation.InsertOrReplace(msg)).ConfigureAwait(false);
}
// Remove.
await _poisonTable.ExecuteAsync(TableOperation.Delete(new PoisonMessage(_storagePartitionKey, _storageRowKey)
{
ETag = "*"
})).ConfigureAwait(false);
}
public Task SkipAuditAsync(EventHubs.EventData @event, string exceptionText)
{
lock (_lock)
{
AuditedEvents.Add(@event);
return(Task.CompletedTask);
}
}
/// <summary>
/// Override the key - as the JSON serialized version loses the Type.
/// </summary>
private static Beef.Events.EventData OverrideKey(Beef.Events.EventData ed, EventHubs.EventData eh)
{
if (eh.Properties.TryGetValue(KeyPropertyName, out var key))
{
ed.Key = key;
}
return(ed);
}
public Task RemoveAsync(EventHubs.EventData @event, PoisonMessageAction action)
{
lock (_lock)
{
RemovedEvents.Add(@event);
Result = PoisonMessageAction.NotPoison;
return(Task.CompletedTask);
}
}
public Task SetAsync(EventHubs.EventData @event, Exception exception)
{
lock (_lock)
{
AddedEvents.Add(@event);
Result = PoisonMessageAction.PoisonRetry;
return(Task.CompletedTask);
}
}
/// <summary>
/// Converts the <see cref="EventHubs.EventData"/> instance to the corresponding <see cref="Beef.Events.EventData{T}"/>.
/// </summary>
/// <typeparam name="T">The value <see cref="Type"/>.</typeparam>
/// <param name="eventData">The <see cref="EventHubs.EventData"/>.</param>
/// <returns>The <see cref="Beef.Events.EventData{T}"/>.</returns>
public static Beef.Events.EventData <T> ToBeefEventData <T>(this EventHubs.EventData eventData)
{
if (eventData == null)
{
throw new ArgumentNullException(nameof(eventData));
}
var body = Encoding.UTF8.GetString(eventData.Body.Array, eventData.Body.Offset, eventData.Body.Count);
return(Newtonsoft.Json.JsonConvert.DeserializeObject <Beef.Events.EventData <T> >(body));
}
/// <summary>
/// Performs the receive processing for an <see cref="EventHubs.EventData"/> instance.
/// </summary>
/// <param name="event">The <see cref="EventHubs.EventData"/> instance to receive/process.</param>
public async Task ReceiveAsync(EventHubs.EventData @event)
{
if (@event == null)
{
return;
}
var(subject, action, _) = @event.GetBeefMetadata();
await ReceiveAsync(subject, action, (subscriber) => subscriber.ValueType == null? @event.ToBeefEventData() : @event.ToBeefEventData(subscriber.ValueType)).ConfigureAwait(false);
}
/// <summary>
/// Converts the <see cref="EventHubs.EventData"/> instance to the corresponding <paramref name="valueType"/> <see cref="Beef.Events.EventData{T}"/>.
/// </summary>
/// <param name="eventData">The <see cref="EventHubs.EventData"/>.</param>
/// <param name="valueType">The value <see cref="Type"/>.</param>
/// <returns>The <see cref="Beef.Events.EventData{T}"/>.</returns>
public static Beef.Events.EventData ToBeefEventData(this EventHubs.EventData eventData, Type valueType)
{
if (eventData == null)
{
throw new ArgumentNullException(nameof(eventData));
}
Beef.Check.NotNull(valueType, nameof(valueType));
var body = Encoding.UTF8.GetString(eventData.Body.Array, eventData.Body.Offset, eventData.Body.Count);
return((Beef.Events.EventData)Newtonsoft.Json.JsonConvert.DeserializeObject(body, typeof(Beef.Events.EventData <>).MakeGenericType(new Type[] { valueType })));
}
/// <summary>
/// Create (instantiate) the <see cref="PoisonMessage"/>.
/// </summary>
private PoisonMessage CreatePoisonMessage(EventHubs.EventData @event, string exceptionText)
{
return(new PoisonMessage(_storagePartitionKey, _storageRowKey)
{
Offset = @event.SystemProperties.Offset,
SequenceNumber = @event.SystemProperties.SequenceNumber,
EnqueuedTimeUtc = @event.SystemProperties.EnqueuedTimeUtc,
PoisonedTimeUtc = DateTime.UtcNow,
FunctionType = _args.Options !.FunctionType,
FunctionName = _args.Options !.FunctionName,
Body = Substring(Encoding.UTF8.GetString(@event.Body.Array)),
Exception = exceptionText
});
/// <summary>
/// A <i>potential</i> poisoned <see cref="EventHubs.EventData"/> has been identified and needs to be orchestrated.
/// </summary>
/// <param name="event">The corresponding <see cref="EventHubs.EventData"/>.</param>
/// <param name="exception">The corresponding <see cref="Exception"/>.</param>
/// <remarks>A <see cref="PoisonMessage"/> will be written to Azure table storage.</remarks>
public async Task SetAsync(EventHubs.EventData @event, Exception exception)
{
if (@event == null)
{
throw new ArgumentNullException(nameof(@event));
}
if (exception == null)
{
throw new ArgumentNullException(nameof(exception));
}
var msg = CreatePoisonMessage(@event, exception.ToString());
await _poisonTable.ExecuteAsync(TableOperation.InsertOrReplace(msg)).ConfigureAwait(false);
}
private static EventHubs.EventData CreateEventData(string offset, long seqNo)
{
var e = new EventHubs.EventData(Encoding.UTF8.GetBytes(offset));
var type = typeof(EventHubs.EventData);
var pi = type.GetProperty("SystemProperties");
pi.SetValue(e, Activator.CreateInstance(pi.PropertyType, true));
var dict = (Dictionary <string, object>)e.SystemProperties;
dict.Add("x-opt-enqueued-time", DateTime.UtcNow);
dict.Add("x-opt-offset", offset);
dict.Add("x-opt-sequence-number", seqNo);
return(e);
}
/// <summary>
/// Gets the <i>Beef</i>-related metadata from the <see cref="EventHubs.EventData"/>.
/// </summary>
/// <param name="eventData">The <see cref="EventHubs.EventData"/>.</param>
/// <returns>The values of the following properties: <see cref="SubjectPropertyName"/>, <see cref="ActionPropertyName"/> and <see cref="TenantIdPropertyName"/>.</returns>
public static (string?subject, string?action, Guid?tenantId) GetBeefMetadata(this EventHubs.EventData eventData)
{
if (eventData == null)
{
throw new ArgumentNullException(nameof(eventData));
}
eventData.Properties.TryGetValue(SubjectPropertyName, out var subject);
eventData.Properties.TryGetValue(ActionPropertyName, out var action);
if (eventData.Properties.TryGetValue(TenantIdPropertyName, out var tenantId) && tenantId != null && tenantId is Guid?)
{
return((string)subject, (string)action, (Guid?)tenantId);
}
else
{
return((string)subject, (string)action, (Guid?)null);
}
}
private async Task <bool> ProcessEventDataAsync(Microsoft.Azure.EventHubs.EventData ehEvent, CancellationToken cancellationToken)
{
try
{
//put business data processing here
return(true);
}
catch (Exception e)
{
//ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService ProcessEventDataAsync met exception= ( {e} )");
string err = $"RouterService ProcessEventDataAsync met exception, exception type={e.GetType().Name}, exception= {e.Message}, at partition ={Context.PartitionId}.";
//ServiceEventSource.Current.Error("RouterService", err);
await HandleExceptionInsideUTask(cancellationToken, e);
throw;
}
}
/// <summary>
/// Perform the checkpoint.
/// </summary>
private async Task CheckpointAsync(PartitionContext context, EventHubs.EventData @event)
{
// Make sure the checkpoint is resulting in a change; avoid the storage perf cost.
if (@event == null || (_lastCheckpoint != null && @event == _lastCheckpoint))
{
return;
}
_lastCheckpoint = @event;
_lastEventData = null;
if (Checkpointer == null)
{
await context.CheckpointAsync(_lastCheckpoint).ConfigureAwait(false);
}
else
{
await Checkpointer(context, @event).ConfigureAwait(false);
}
}
/// <summary>
/// Performs the receive processing for an <see cref="EventHubs.EventData"/> instance.
/// </summary>
/// <param name="event">The <see cref="EventHubs.EventData"/> instance to receive/process.</param>
public async Task ReceiveAsync(EventHubs.EventData @event)
{
if (@event == null)
{
return;
}
// Convert EventHubs.EventData to Beef.EventData..
var(subject, action, _) = @event.GetBeefMetadata();
await ReceiveAsync(subject, action, (subscriber) =>
{
try
{
return(subscriber.ValueType == null ? @event.ToBeefEventData() : @event.ToBeefEventData(subscriber.ValueType));
}
#pragma warning disable CA1031 // Do not catch general exception types; by design, need this to be a catch-all.
catch (Exception ex) { throw new InvalidEventDataException(ex); }
#pragma warning restore CA1031
}).ConfigureAwait(false);
}
/// <summary>
/// Converts the <see cref="Beef.Events.EventData"/> to a corresponding <see cref="EventHubs.EventData"/>.
/// </summary>
/// <param name="eventData">The <see cref="EventHubs.EventData"/>.</param>
/// <returns>The <see cref="Beef.Events.EventData"/>.</returns>
public static EventHubs.EventData ToEventHubsEventData(this Beef.Events.EventData eventData)
{
if (eventData == null)
{
throw new ArgumentNullException(nameof(eventData));
}
if (string.IsNullOrEmpty(eventData.Subject))
{
throw new ArgumentException("Subject property is required to be set.", nameof(eventData));
}
var json = JsonConvert.SerializeObject(eventData);
var bytes = Encoding.UTF8.GetBytes(json);
var ed = new EventHubs.EventData(bytes);
ed.Properties.Add(SubjectPropertyName, eventData.Subject);
ed.Properties.Add(ActionPropertyName, eventData.Action);
ed.Properties.Add(TenantIdPropertyName, eventData.TenantId);
return(ed);
}
/// <summary>
/// Gets the <i>Beef</i>-related metadata from the <see cref="EventHubs.EventData"/>.
/// </summary>
/// <param name="eventData">The <see cref="EventHubs.EventData"/>.</param>
/// <returns>The values of the following properties: <see cref="SubjectPropertyName"/>, <see cref="ActionPropertyName"/> and <see cref="TenantIdPropertyName"/>.</returns>
public static (string subject, string action, Guid?tenantId) GetBeefMetadata(this EventHubs.EventData eventData)
{
if (eventData == null)
{
throw new ArgumentNullException(nameof(eventData));
}
if (!eventData.Properties.TryGetValue(SubjectPropertyName, out var subject) || string.IsNullOrEmpty((string)subject))
{
throw new ArgumentException($"EventData does not contain required property '{SubjectPropertyName}'.");
}
eventData.Properties.TryGetValue(ActionPropertyName, out var action);
if (eventData.Properties.TryGetValue(TenantIdPropertyName, out var tenantId) && tenantId != null && tenantId is Guid?)
{
return((string)subject, (string)action, (Guid?)tenantId);
}
else
{
return((string)subject, (string)action, (Guid?)null);
}
}
static void Main(string[] args)
{
string connectionString = ConfigurationSettings.AppSettings["EventHubConnection"];
var connectionStringBuilder = new EventHubsConnectionStringBuilder(connectionString)
{
EntityPath = "rafat-eventhub1"
};
var client = Microsoft.Azure.EventHubs.EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
int i = 0;
while (i++ < 1000000)
{
Random rnd = new Random(i);
int _temp = rnd.Next(100);
string json = string.Format("{{\"iotid\":\"{0}\",\"temp\":{1}}}", GetRandomlyRepeatedString(3), _temp);
EventData data = new Microsoft.Azure.EventHubs.EventData(Encoding.UTF8.GetBytes(json));
client.SendAsync(data);
System.Threading.Thread.Sleep(100);
Console.Write(".");
}
Console.WriteLine("Done sending 100 events");
Console.Read();
}
/// <summary> /// Initializes a new instance of the <see cref="ResilientEventHubData"/> class. /// </summary> /// <param name="eventData"></param> public ResilientEventHubData(EventHubs.EventData eventData) => EventData = eventData;
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
// Get a reference to the current connections array first, just in case there is another thread wanting to clean
// up the connections with CleanUpAsync(), we won't get a null reference exception here.
EventHubClient[] currentClients = Interlocked.CompareExchange <EventHubClient[]>(ref this.clients, this.clients, this.clients);
if (currentClients == null || events == null || events.Count == 0)
{
return;
}
try
{
// Since event hub limits each message/batch to be a certain size, we need to
// keep checking the size for exceeds and split into a new batch as needed
List <List <MessagingEventData> > batches = new List <List <MessagingEventData> >();
int batchByteSize = 0;
foreach (EventData eventData in events)
{
int messageSize;
MessagingEventData messagingEventData = eventData.ToMessagingEventData(out messageSize);
// If we don't have a batch yet, or the addition of this message will exceed the limit for this batch, then
// start a new batch.
if (batches.Count == 0 ||
batchByteSize + messageSize > EventHubMessageSizeLimit)
{
batches.Add(new List <MessagingEventData>());
batchByteSize = 0;
}
batchByteSize += messageSize;
List <MessagingEventData> currentBatch = batches[batches.Count - 1];
currentBatch.Add(messagingEventData);
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
EventHubClient hubClient = currentClients[transmissionSequenceNumber % ConcurrentConnections];
List <Task> tasks = new List <Task>();
foreach (List <MessagingEventData> batch in batches)
{
tasks.Add(hubClient.SendAsync(batch));
}
await Task.WhenAll(tasks).ConfigureAwait(false);
this.healthReporter.ReportHealthy();
}
catch (Exception e)
{
string errorMessage = nameof(EventHubOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
this.healthReporter.ReportProblem(errorMessage);
}
}
/// <summary>
/// Process the events.
/// </summary>
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventHubs.EventData> events)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (events == null)
{
throw new ArgumentNullException(nameof(events));
}
// Note that checkpointing is only performed where an error occurs, and at the end of processing all events, to minimise chattiness to storage.
try
{
// Convert to an array and make sure there is work to do.
var array = events.ToArray();
if (array.Length == 0)
{
return;
}
// Reset "last" as we are executing a new set of events.
_lastCheckpoint = _lastEventData = null;
// Process all events (note: we want to minimise the checkpointing, so will only do when all processed, or where we encounter an error to retry.
for (int i = 0; i < array.Length; i++)
{
// Stop where a cancel has been requested.
if (context.CancellationToken.IsCancellationRequested)
{
break;
}
// Cancellation token passed into polly, as well as the function, as either may need to cancel on request.
_currEventData = array[i];
await _policyAsync.ExecuteAsync(async (ct) => await ExecuteCurrentEvent(context, ct).ConfigureAwait(false), context.CancellationToken).ConfigureAwait(false);
// Remember, remember the 5th of November (https://www.youtube.com/watch?v=LF1951pENdk) and the last event data that was successful.
_lastEventData = _currEventData;
_currEventData = null;
}
// Array complete (or cancelled), so checkpoint on the last.
await CheckpointAsync(context, _lastEventData).ConfigureAwait(false);
_logger.LogInformation($"Batch of '{array.Length}' event(s) completed {GetPartitionContextLogInfo(context)}.");
}
catch (TaskCanceledException) { throw; } // Expected; carry on.
catch (EventHubs.ReceiverDisconnectedException) { throw; } // Expected; carry on.
catch (LeaseLostException) { throw; } // Expected; carry on.
catch (Exception ex) // Catch all, log, and carry on.
{
_logger.LogCritical(ex, $"Unexpected/unhandled exception occured during processing {GetPartitionContextLogInfo(context)}: {ex.Message}");
throw;
}
finally
{
// Dispose all of the events as we are done with them (regardless of whether all processed).
foreach (var e in events)
{
e.Dispose();
}
}
}
public static async Task Run([ResilientEventHubTrigger] EventHubs.EventData @event, ILogger log)
{
await EventHubSubscriberHost.Create(log).ReceiveAsync(@event);
}
public async Task SendMessage <T>(T data)
{
var inData = JsonConvert.SerializeObject(data);
var message = new EventData(Encoding.UTF8.GetBytes(inData));
await client.SendAsync(message);
}
public Task <PoisonMessageAction> CheckAsync(EventHubs.EventData @event) => Task.FromResult(Result);
/// <summary>
/// Gets the standard log information for a <see cref="PartitionContext"/>.
/// </summary>
private static string GetEventDataLogInfo(PartitionContext context, EventHubs.EventData @event) => $"[PartitionId: '{context.PartitionId}', Offset: '{@event.SystemProperties?.Offset}', SequenceNumber: '{@event.SystemProperties?.SequenceNumber}']";
