/// <summary> /// A <i>potential</i> poisoned <see cref="EventHubs.EventData"/> has been identified and needs to be orchestrated. /// </summary> /// <param name="event">The corresponding <see cref="EventHubs.EventData"/>.</param> /// <param name="exception">The corresponding <see cref="Exception"/>.</param> /// <remarks>A <see cref="PoisonMessage"/> will be written to Azure table storage.</remarks> public async Task SetAsync(EventHubs.EventData @event, Exception exception) { if (@event == null) { throw new ArgumentNullException(nameof(@event)); } if (exception == null) { throw new ArgumentNullException(nameof(exception)); } var msg = new PoisonMessage(_storagePartitionKey, _storageRowKey) { Offset = @event.SystemProperties.Offset, SequenceNumber = @event.SystemProperties.SequenceNumber, EnqueuedTimeUtc = @event.SystemProperties.EnqueuedTimeUtc, PoisonedTimeUtc = DateTime.UtcNow, FunctionType = _args.Options !.FunctionType, FunctionName = _args.Options !.FunctionName, Body = Substring(Encoding.UTF8.GetString(@event.Body.Array)), Exception = Substring(exception.ToString()) }; await _poisonTable.ExecuteAsync(TableOperation.InsertOrReplace(msg)).ConfigureAwait(false); }
/// <summary> /// Checks whether the <see cref="EventHubs.EventData"/> is in a <b>Poison</b> state and determines the corresponding <see cref="PoisonMessageAction"/>. /// </summary> /// <param name="event">The corresponding <see cref="EventData"/>.</param> /// <returns>The resulting <see cref="PoisonMessageAction"/>.</returns> /// <remarks>Reads the <see cref="PoisonMessage"/> in Azure table storage to determine the result. Where <see cref="PoisonMessage.SkipMessage"/> is <c>true</c>, then /// <see cref="PoisonMessageAction.PoisonSkip"/> will be returned; otherwise, <see cref="PoisonMessageAction.PoisonRetry"/>.</remarks> public async Task <PoisonMessageAction> CheckAsync(EventHubs.EventData @event) { if (@event == null) { throw new ArgumentNullException(nameof(@event)); } var msg = await GetPoisonMessageAsync().ConfigureAwait(false); if (msg == null) { return(PoisonMessageAction.NotPoison); } if (@event.SystemProperties.SequenceNumber != msg.SequenceNumber) { // Warn if event exists with different offset - this means things are slightly out of whack! _args.Logger.LogWarning($"EventData (Seq#: '{@event.SystemProperties.SequenceNumber}') being processed is out of sync with persisted Poison Message (Seq#: '{msg.SequenceNumber}'); EventData assumed correct and Poison Message deleted."); await RemoveAsync(@event, PoisonMessageAction.Undetermined).ConfigureAwait(false); } if (msg.SkipMessage) { return(PoisonMessageAction.PoisonSkip); } else { return(PoisonMessageAction.PoisonRetry); } }
public static EventHubs.EventData CreateEvent(int exceptionCount) { var e = new EventHubs.EventData(Encoding.UTF8.GetBytes(Entities.Cleaner.Clean(DateTime.Now).ToString())); e.Properties.Add("ExceptionCount", exceptionCount); return(e); }
public static void Run([CosmosDBTrigger( databaseName: "IoT", collectionName: "IoT1", ConnectionStringSetting = "DBConnection", LeaseCollectionName = "leases")] IReadOnlyList <Document> input, TraceWriter log) { string connectionString = ConfigurationSettings.AppSettings["EventHubConnection"]; var connectionStringBuilder = new EventHubsConnectionStringBuilder(connectionString) { EntityPath = "rafat-eventhub1" }; var client = Microsoft.Azure.EventHubs.EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString()); foreach (var doc in input) { string json = string.Format("{{\"iotid\":\"{0}\",\"temp\":{1}}}", doc.GetPropertyValue <string>("iotid"), doc.GetPropertyValue <string>("temp")); EventData data = new Microsoft.Azure.EventHubs.EventData(Encoding.UTF8.GetBytes(json)); client.SendAsync(data); } if (input != null && input.Count > 0) { log.Verbose("Documents modified " + input.Count); log.Verbose("First document Id " + input[0].Id); } }
/// <summary> /// Publishes the <paramref name="events"/>. /// </summary> /// <param name="events">The <see cref="EventData"/> instances.</param> /// <returns>The corresponding <see cref="Task"/>.</returns> protected override async Task PublishEventsAsync(params EventData[] events) { if (events == null || events.Length == 0) { return; } var partitionKey = GetPartitionKey(events); var eventHubEvents = new EventHubs.EventData[events.Length]; for (int i = 0; i < events.Length; i++) { eventHubEvents[i] = events[i].ToEventHubsEventData(); } try { await _invoker.InvokeAsync(this, async() => await _client.SendAsync(eventHubEvents, partitionKey).ConfigureAwait(false)).ConfigureAwait(false); } #pragma warning disable CA1031 // Do not catch general exception types; by-design, is a catch all. catch (Exception ex) #pragma warning restore CA1031 { OnException(events, partitionKey, ex); if (!SwallowException) { throw; } } }
/// <summary> /// A previously identified poisoned <see cref="EventHubs.EventData"/> has either successfully processed or can be skipped and should be removed. /// </summary> /// <param name="event">The corresponding <see cref="EventHubs.EventData"/>.</param> /// <param name="action">The corresponding reason (<see cref="PoisonMessageAction"/>) for removal.</param> /// <remarks>The corresponding <see cref="PoisonMessage"/> will be removed/deleted from Azure table storage.</remarks> public async Task RemoveAsync(EventHubs.EventData @event, PoisonMessageAction action) { if (@event == null) { throw new ArgumentNullException(nameof(@event)); } var msg = await GetPoisonMessageAsync().ConfigureAwait(false); if (msg == null) { return; } // Audit the skipped record. if (action == PoisonMessageAction.PoisonSkip) { msg.SkippedTimeUtc = DateTime.UtcNow; msg.RowKey = msg.SkippedTimeUtc.Value.ToString("o", System.Globalization.CultureInfo.InvariantCulture) + "-" + msg.RowKey; await _skippedTable.ExecuteAsync(TableOperation.InsertOrReplace(msg)).ConfigureAwait(false); } // Remove. await _poisonTable.ExecuteAsync(TableOperation.Delete(new PoisonMessage(_storagePartitionKey, _storageRowKey) { ETag = "*" })).ConfigureAwait(false); }
public Task SkipAuditAsync(EventHubs.EventData @event, string exceptionText) { lock (_lock) { AuditedEvents.Add(@event); return(Task.CompletedTask); } }
/// <summary> /// Override the key - as the JSON serialized version loses the Type. /// </summary> private static Beef.Events.EventData OverrideKey(Beef.Events.EventData ed, EventHubs.EventData eh) { if (eh.Properties.TryGetValue(KeyPropertyName, out var key)) { ed.Key = key; } return(ed); }
public Task RemoveAsync(EventHubs.EventData @event, PoisonMessageAction action) { lock (_lock) { RemovedEvents.Add(@event); Result = PoisonMessageAction.NotPoison; return(Task.CompletedTask); } }
public Task SetAsync(EventHubs.EventData @event, Exception exception) { lock (_lock) { AddedEvents.Add(@event); Result = PoisonMessageAction.PoisonRetry; return(Task.CompletedTask); } }
/// <summary> /// Converts the <see cref="EventHubs.EventData"/> instance to the corresponding <see cref="Beef.Events.EventData{T}"/>. /// </summary> /// <typeparam name="T">The value <see cref="Type"/>.</typeparam> /// <param name="eventData">The <see cref="EventHubs.EventData"/>.</param> /// <returns>The <see cref="Beef.Events.EventData{T}"/>.</returns> public static Beef.Events.EventData <T> ToBeefEventData <T>(this EventHubs.EventData eventData) { if (eventData == null) { throw new ArgumentNullException(nameof(eventData)); } var body = Encoding.UTF8.GetString(eventData.Body.Array, eventData.Body.Offset, eventData.Body.Count); return(Newtonsoft.Json.JsonConvert.DeserializeObject <Beef.Events.EventData <T> >(body)); }
/// <summary> /// Performs the receive processing for an <see cref="EventHubs.EventData"/> instance. /// </summary> /// <param name="event">The <see cref="EventHubs.EventData"/> instance to receive/process.</param> public async Task ReceiveAsync(EventHubs.EventData @event) { if (@event == null) { return; } var(subject, action, _) = @event.GetBeefMetadata(); await ReceiveAsync(subject, action, (subscriber) => subscriber.ValueType == null? @event.ToBeefEventData() : @event.ToBeefEventData(subscriber.ValueType)).ConfigureAwait(false); }
/// <summary> /// Converts the <see cref="EventHubs.EventData"/> instance to the corresponding <paramref name="valueType"/> <see cref="Beef.Events.EventData{T}"/>. /// </summary> /// <param name="eventData">The <see cref="EventHubs.EventData"/>.</param> /// <param name="valueType">The value <see cref="Type"/>.</param> /// <returns>The <see cref="Beef.Events.EventData{T}"/>.</returns> public static Beef.Events.EventData ToBeefEventData(this EventHubs.EventData eventData, Type valueType) { if (eventData == null) { throw new ArgumentNullException(nameof(eventData)); } Beef.Check.NotNull(valueType, nameof(valueType)); var body = Encoding.UTF8.GetString(eventData.Body.Array, eventData.Body.Offset, eventData.Body.Count); return((Beef.Events.EventData)Newtonsoft.Json.JsonConvert.DeserializeObject(body, typeof(Beef.Events.EventData <>).MakeGenericType(new Type[] { valueType }))); }
/// <summary> /// Create (instantiate) the <see cref="PoisonMessage"/>. /// </summary> private PoisonMessage CreatePoisonMessage(EventHubs.EventData @event, string exceptionText) { return(new PoisonMessage(_storagePartitionKey, _storageRowKey) { Offset = @event.SystemProperties.Offset, SequenceNumber = @event.SystemProperties.SequenceNumber, EnqueuedTimeUtc = @event.SystemProperties.EnqueuedTimeUtc, PoisonedTimeUtc = DateTime.UtcNow, FunctionType = _args.Options !.FunctionType, FunctionName = _args.Options !.FunctionName, Body = Substring(Encoding.UTF8.GetString(@event.Body.Array)), Exception = exceptionText });
/// <summary> /// A <i>potential</i> poisoned <see cref="EventHubs.EventData"/> has been identified and needs to be orchestrated. /// </summary> /// <param name="event">The corresponding <see cref="EventHubs.EventData"/>.</param> /// <param name="exception">The corresponding <see cref="Exception"/>.</param> /// <remarks>A <see cref="PoisonMessage"/> will be written to Azure table storage.</remarks> public async Task SetAsync(EventHubs.EventData @event, Exception exception) { if (@event == null) { throw new ArgumentNullException(nameof(@event)); } if (exception == null) { throw new ArgumentNullException(nameof(exception)); } var msg = CreatePoisonMessage(@event, exception.ToString()); await _poisonTable.ExecuteAsync(TableOperation.InsertOrReplace(msg)).ConfigureAwait(false); }
private static EventHubs.EventData CreateEventData(string offset, long seqNo) { var e = new EventHubs.EventData(Encoding.UTF8.GetBytes(offset)); var type = typeof(EventHubs.EventData); var pi = type.GetProperty("SystemProperties"); pi.SetValue(e, Activator.CreateInstance(pi.PropertyType, true)); var dict = (Dictionary <string, object>)e.SystemProperties; dict.Add("x-opt-enqueued-time", DateTime.UtcNow); dict.Add("x-opt-offset", offset); dict.Add("x-opt-sequence-number", seqNo); return(e); }
/// <summary> /// Gets the <i>Beef</i>-related metadata from the <see cref="EventHubs.EventData"/>. /// </summary> /// <param name="eventData">The <see cref="EventHubs.EventData"/>.</param> /// <returns>The values of the following properties: <see cref="SubjectPropertyName"/>, <see cref="ActionPropertyName"/> and <see cref="TenantIdPropertyName"/>.</returns> public static (string?subject, string?action, Guid?tenantId) GetBeefMetadata(this EventHubs.EventData eventData) { if (eventData == null) { throw new ArgumentNullException(nameof(eventData)); } eventData.Properties.TryGetValue(SubjectPropertyName, out var subject); eventData.Properties.TryGetValue(ActionPropertyName, out var action); if (eventData.Properties.TryGetValue(TenantIdPropertyName, out var tenantId) && tenantId != null && tenantId is Guid?) { return((string)subject, (string)action, (Guid?)tenantId); } else { return((string)subject, (string)action, (Guid?)null); } }
private async Task <bool> ProcessEventDataAsync(Microsoft.Azure.EventHubs.EventData ehEvent, CancellationToken cancellationToken) { try { //put business data processing here return(true); } catch (Exception e) { //ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService ProcessEventDataAsync met exception= ( {e} )"); string err = $"RouterService ProcessEventDataAsync met exception, exception type={e.GetType().Name}, exception= {e.Message}, at partition ={Context.PartitionId}."; //ServiceEventSource.Current.Error("RouterService", err); await HandleExceptionInsideUTask(cancellationToken, e); throw; } }
/// <summary> /// Perform the checkpoint. /// </summary> private async Task CheckpointAsync(PartitionContext context, EventHubs.EventData @event) { // Make sure the checkpoint is resulting in a change; avoid the storage perf cost. if (@event == null || (_lastCheckpoint != null && @event == _lastCheckpoint)) { return; } _lastCheckpoint = @event; _lastEventData = null; if (Checkpointer == null) { await context.CheckpointAsync(_lastCheckpoint).ConfigureAwait(false); } else { await Checkpointer(context, @event).ConfigureAwait(false); } }
/// <summary> /// Performs the receive processing for an <see cref="EventHubs.EventData"/> instance. /// </summary> /// <param name="event">The <see cref="EventHubs.EventData"/> instance to receive/process.</param> public async Task ReceiveAsync(EventHubs.EventData @event) { if (@event == null) { return; } // Convert EventHubs.EventData to Beef.EventData.. var(subject, action, _) = @event.GetBeefMetadata(); await ReceiveAsync(subject, action, (subscriber) => { try { return(subscriber.ValueType == null ? @event.ToBeefEventData() : @event.ToBeefEventData(subscriber.ValueType)); } #pragma warning disable CA1031 // Do not catch general exception types; by design, need this to be a catch-all. catch (Exception ex) { throw new InvalidEventDataException(ex); } #pragma warning restore CA1031 }).ConfigureAwait(false); }
/// <summary> /// Converts the <see cref="Beef.Events.EventData"/> to a corresponding <see cref="EventHubs.EventData"/>. /// </summary> /// <param name="eventData">The <see cref="EventHubs.EventData"/>.</param> /// <returns>The <see cref="Beef.Events.EventData"/>.</returns> public static EventHubs.EventData ToEventHubsEventData(this Beef.Events.EventData eventData) { if (eventData == null) { throw new ArgumentNullException(nameof(eventData)); } if (string.IsNullOrEmpty(eventData.Subject)) { throw new ArgumentException("Subject property is required to be set.", nameof(eventData)); } var json = JsonConvert.SerializeObject(eventData); var bytes = Encoding.UTF8.GetBytes(json); var ed = new EventHubs.EventData(bytes); ed.Properties.Add(SubjectPropertyName, eventData.Subject); ed.Properties.Add(ActionPropertyName, eventData.Action); ed.Properties.Add(TenantIdPropertyName, eventData.TenantId); return(ed); }
/// <summary> /// Gets the <i>Beef</i>-related metadata from the <see cref="EventHubs.EventData"/>. /// </summary> /// <param name="eventData">The <see cref="EventHubs.EventData"/>.</param> /// <returns>The values of the following properties: <see cref="SubjectPropertyName"/>, <see cref="ActionPropertyName"/> and <see cref="TenantIdPropertyName"/>.</returns> public static (string subject, string action, Guid?tenantId) GetBeefMetadata(this EventHubs.EventData eventData) { if (eventData == null) { throw new ArgumentNullException(nameof(eventData)); } if (!eventData.Properties.TryGetValue(SubjectPropertyName, out var subject) || string.IsNullOrEmpty((string)subject)) { throw new ArgumentException($"EventData does not contain required property '{SubjectPropertyName}'."); } eventData.Properties.TryGetValue(ActionPropertyName, out var action); if (eventData.Properties.TryGetValue(TenantIdPropertyName, out var tenantId) && tenantId != null && tenantId is Guid?) { return((string)subject, (string)action, (Guid?)tenantId); } else { return((string)subject, (string)action, (Guid?)null); } }
static void Main(string[] args) { string connectionString = ConfigurationSettings.AppSettings["EventHubConnection"]; var connectionStringBuilder = new EventHubsConnectionStringBuilder(connectionString) { EntityPath = "rafat-eventhub1" }; var client = Microsoft.Azure.EventHubs.EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString()); int i = 0; while (i++ < 1000000) { Random rnd = new Random(i); int _temp = rnd.Next(100); string json = string.Format("{{\"iotid\":\"{0}\",\"temp\":{1}}}", GetRandomlyRepeatedString(3), _temp); EventData data = new Microsoft.Azure.EventHubs.EventData(Encoding.UTF8.GetBytes(json)); client.SendAsync(data); System.Threading.Thread.Sleep(100); Console.Write("."); } Console.WriteLine("Done sending 100 events"); Console.Read(); }
/// <summary> /// Initializes a new instance of the <see cref="ResilientEventHubData"/> class. /// </summary> /// <param name="eventData"></param> public ResilientEventHubData(EventHubs.EventData eventData) => EventData = eventData;
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken) { // Get a reference to the current connections array first, just in case there is another thread wanting to clean // up the connections with CleanUpAsync(), we won't get a null reference exception here. EventHubClient[] currentClients = Interlocked.CompareExchange <EventHubClient[]>(ref this.clients, this.clients, this.clients); if (currentClients == null || events == null || events.Count == 0) { return; } try { // Since event hub limits each message/batch to be a certain size, we need to // keep checking the size for exceeds and split into a new batch as needed List <List <MessagingEventData> > batches = new List <List <MessagingEventData> >(); int batchByteSize = 0; foreach (EventData eventData in events) { int messageSize; MessagingEventData messagingEventData = eventData.ToMessagingEventData(out messageSize); // If we don't have a batch yet, or the addition of this message will exceed the limit for this batch, then // start a new batch. if (batches.Count == 0 || batchByteSize + messageSize > EventHubMessageSizeLimit) { batches.Add(new List <MessagingEventData>()); batchByteSize = 0; } batchByteSize += messageSize; List <MessagingEventData> currentBatch = batches[batches.Count - 1]; currentBatch.Add(messagingEventData); } if (cancellationToken.IsCancellationRequested) { return; } EventHubClient hubClient = currentClients[transmissionSequenceNumber % ConcurrentConnections]; List <Task> tasks = new List <Task>(); foreach (List <MessagingEventData> batch in batches) { tasks.Add(hubClient.SendAsync(batch)); } await Task.WhenAll(tasks).ConfigureAwait(false); this.healthReporter.ReportHealthy(); } catch (Exception e) { string errorMessage = nameof(EventHubOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString(); this.healthReporter.ReportProblem(errorMessage); } }
/// <summary> /// Process the events. /// </summary> public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventHubs.EventData> events) { if (context == null) { throw new ArgumentNullException(nameof(context)); } if (events == null) { throw new ArgumentNullException(nameof(events)); } // Note that checkpointing is only performed where an error occurs, and at the end of processing all events, to minimise chattiness to storage. try { // Convert to an array and make sure there is work to do. var array = events.ToArray(); if (array.Length == 0) { return; } // Reset "last" as we are executing a new set of events. _lastCheckpoint = _lastEventData = null; // Process all events (note: we want to minimise the checkpointing, so will only do when all processed, or where we encounter an error to retry. for (int i = 0; i < array.Length; i++) { // Stop where a cancel has been requested. if (context.CancellationToken.IsCancellationRequested) { break; } // Cancellation token passed into polly, as well as the function, as either may need to cancel on request. _currEventData = array[i]; await _policyAsync.ExecuteAsync(async (ct) => await ExecuteCurrentEvent(context, ct).ConfigureAwait(false), context.CancellationToken).ConfigureAwait(false); // Remember, remember the 5th of November (https://www.youtube.com/watch?v=LF1951pENdk) and the last event data that was successful. _lastEventData = _currEventData; _currEventData = null; } // Array complete (or cancelled), so checkpoint on the last. await CheckpointAsync(context, _lastEventData).ConfigureAwait(false); _logger.LogInformation($"Batch of '{array.Length}' event(s) completed {GetPartitionContextLogInfo(context)}."); } catch (TaskCanceledException) { throw; } // Expected; carry on. catch (EventHubs.ReceiverDisconnectedException) { throw; } // Expected; carry on. catch (LeaseLostException) { throw; } // Expected; carry on. catch (Exception ex) // Catch all, log, and carry on. { _logger.LogCritical(ex, $"Unexpected/unhandled exception occured during processing {GetPartitionContextLogInfo(context)}: {ex.Message}"); throw; } finally { // Dispose all of the events as we are done with them (regardless of whether all processed). foreach (var e in events) { e.Dispose(); } } }
public static async Task Run([ResilientEventHubTrigger] EventHubs.EventData @event, ILogger log) { await EventHubSubscriberHost.Create(log).ReceiveAsync(@event); }
public async Task SendMessage <T>(T data) { var inData = JsonConvert.SerializeObject(data); var message = new EventData(Encoding.UTF8.GetBytes(inData)); await client.SendAsync(message); }
public Task <PoisonMessageAction> CheckAsync(EventHubs.EventData @event) => Task.FromResult(Result);
/// <summary> /// Gets the standard log information for a <see cref="PartitionContext"/>. /// </summary> private static string GetEventDataLogInfo(PartitionContext context, EventHubs.EventData @event) => $"[PartitionId: '{context.PartitionId}', Offset: '{@event.SystemProperties?.Offset}', SequenceNumber: '{@event.SystemProperties?.SequenceNumber}']";