/// <summary>
/// Serializes the object to JSON.
/// </summary>
/// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param>
/// <param name="obj">The object to serialize to JSON.</param>
internal static void Serialize(JsonWriter writer, PartitionEvent obj)
{
// Required properties are always serialized, optional properties are serialized when not null.
writer.WriteStartObject();
writer.WriteProperty(obj.Kind.ToString(), "Kind", JsonWriterExtensions.WriteStringValue);
writer.WriteProperty(obj.EventInstanceId, "EventInstanceId", JsonWriterExtensions.WriteGuidValue);
writer.WriteProperty(obj.TimeStamp, "TimeStamp", JsonWriterExtensions.WriteDateTimeValue);
writer.WriteProperty(obj.PartitionId, "PartitionId", PartitionIdConverter.Serialize);
if (obj.HasCorrelatedEvents != null)
{
writer.WriteProperty(obj.HasCorrelatedEvents, "HasCorrelatedEvents", JsonWriterExtensions.WriteBoolValue);
}
writer.WriteEndObject();
}
private static dynamic AddMetaData(int count, PartitionEvent receivedEvent, string data, string partition, long offset, long sequence)
{
dynamic message = JsonConvert.DeserializeObject(data);
message.AzureEventHubsPartition = partition;
message.AzureEventHubsOffset = offset;
message.AzureEventHubsSequence = sequence;
message.AzureEnqueuedTime = receivedEvent.Data.EnqueuedTime.ToString("o");
if (count == 0)
{
Console.WriteLine($"First Message EnqueueTime: {message.AzureEnqueuedTime}, Offset: {message.AzureEventHubsOffset}, Sequence: {message.AzureEventHubsSequence}");
}
return(message);
}
public void DisplayMessage(PartitionEvent pe)
{
var payload = Encoding.UTF8.GetString(pe.Data.Body.ToArray());
var outputMessage = $@"{{
""body"": {FormatPayloadField(payload)},
""enqueuedTime"": ""{pe.Data.EnqueuedTime}"",
""systemProperties"": {{
{pe.Data.SystemProperties.Count}
}}
""properties"": {{
{pe.Data.Properties.Count}
}}
}}";
var sb = new StringBuilder();
var padding1 = " ";
var padding2 = " ";
sb.AppendLine("{");
sb.Append(padding1 + "\"body\": ");
sb.Append(FormatPayloadField(payload));
sb.AppendLine(",");
sb.AppendLine(padding1 + "\"systemProperties\": {");
foreach (var kv in pe.Data.SystemProperties)
{
sb.AppendLine(padding2 + $"\"{kv.Key}\": {FormatPayloadField(kv.Value.ToString())}");
}
sb.AppendLine(padding1 + "}");
sb.AppendLine(padding1 + "\"properties\": {");
foreach (var kv in pe.Data.Properties)
{
sb.AppendLine(padding2 + $"\"{kv.Key}\": \"{kv.Value}\"");
}
sb.AppendLine(padding1 + "}");
sb.AppendLine("}");
Console.WriteLine(sb);
}
public async Task ReadAllPartitions()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample07_ReadAllPartitions
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = scope.EventHubName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using (CancellationTokenSource cancellationSource = new CancellationTokenSource())
{
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
int eventsRead = 0;
int maximumEvents = 50;
IAsyncEnumerator <PartitionEvent> iterator =
consumer.ReadEventsAsync(cancellationSource.Token).GetAsyncEnumerator();
try
{
while (await iterator.MoveNextAsync())
{
PartitionEvent partitionEvent = iterator.Current;
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
eventsRead++;
if (eventsRead >= maximumEvents)
{
break;
}
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await iterator.DisposeAsync();
}
}
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
/// <summary>
/// Responsible for processing events received from the Event Hubs service.
/// </summary>
///
/// <param name="partitionEvent">The partition event to be processed.</param>
/// <param name="context">The context in which the associated partition is being processed.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
protected abstract ValueTask ProcessEventAsync(PartitionEvent partitionEvent,
T context);
public void SubmitInternalEvent(PartitionEvent evt)
{
this.logWorker.SubmitInternalEvent(evt);
}
// TODO: Update all the logging messages
async Task PartitionEventLoop(long nextPacketToReceive)
{
this.host.logger.LogDebug("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) starting receive loop", this.host.eventHubPath, this.partitionId, this.Incarnation);
try
{
this.partitionReceiver = this.host.connections.CreatePartitionReceiver((int)this.partitionId, this.host.consumerGroupName, nextPacketToReceive);
while (!this.shutdownSource.IsCancellationRequested)
{
this.host.logger.LogTrace("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) trying to receive eventdata from position {position}", this.host.eventHubPath, this.partitionId, this.Incarnation, nextPacketToReceive);
IEnumerable <EventData> eventData;
try
{
var receiveTask = this.partitionReceiver.ReceiveAsync(MaxReceiveBatchSize, TimeSpan.FromMinutes(1));
await Task.WhenAny(receiveTask, this.shutdownTask).ConfigureAwait(false);
this.shutdownSource.Token.ThrowIfCancellationRequested();
eventData = await receiveTask.ConfigureAwait(false);
}
catch (TimeoutException exception)
{
// not sure that we should be seeing this, but we do.
this.host.logger.LogWarning("Retrying after transient(?) TimeoutException in ReceiveAsync {exception}", exception);
eventData = null;
}
if (eventData != null)
{
this.host.logger.LogDebug("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) received eventdata from position {position}", this.host.eventHubPath, this.partitionId, this.Incarnation, nextPacketToReceive);
var batch = new List <PartitionEvent>();
var receivedTimestamp = this.partition.CurrentTimeMs;
foreach (var eventDatum in eventData)
{
var seqno = eventDatum.SystemProperties.SequenceNumber;
if (seqno == nextPacketToReceive)
{
PartitionEvent partitionEvent = null;
try
{
Packet.Deserialize(eventDatum.Body, out partitionEvent, this.host.taskHubGuid);
}
catch (Exception)
{
this.host.logger.LogError("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) could not deserialize packet #{seqno} ({size} bytes)", this.host.eventHubPath, this.partitionId, this.Incarnation, seqno, eventDatum.Body.Count);
throw;
}
nextPacketToReceive = seqno + 1;
if (partitionEvent != null)
{
this.host.logger.LogDebug("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) received packet #{seqno} ({size} bytes) {event}", this.host.eventHubPath, this.partitionId, this.Incarnation, seqno, eventDatum.Body.Count, partitionEvent);
}
else
{
this.host.logger.LogDebug("EventHubsProcessor {eventHubName}/{eventHubPartition}({incarnation}) ignored packet #{seqno} for different taskhub", this.host.eventHubPath, this.partitionId, this.Incarnation, seqno);
continue;
}
partitionEvent.NextInputQueuePosition = nextPacketToReceive;
batch.Add(partitionEvent);
partitionEvent.ReceivedTimestamp = this.partition.CurrentTimeMs;
}
else if (seqno > nextPacketToReceive)
{
this.host.logger.LogError("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) received wrong packet, #{seqno} instead of #{expected}", this.host.eventHubPath, this.partitionId, this.Incarnation, seqno, nextPacketToReceive);
// this should never happen, as EventHubs guarantees in-order delivery of packets
throw new InvalidOperationException("EventHubs Out-Of-Order Packet");
}
else
{
this.host.logger.LogDebug("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) discarded packet #{seqno} because it is already processed", this.host.eventHubPath, this.partitionId, this.Incarnation, seqno);
}
}
if (batch.Count > 0 && !this.shutdownSource.IsCancellationRequested)
{
this.host.logger.LogDebug("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) received batch of {batchsize} packets, starting with #{seqno}, next expected packet is #{nextSeqno}", this.host.eventHubPath, this.partitionId, this.Incarnation, batch.Count, batch[0].NextInputQueuePosition - 1, nextPacketToReceive);
this.partition.SubmitExternalEvents(batch);
}
}
}
}
catch (OperationCanceledException)
{
this.host.logger.LogInformation("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) was terminated", this.host.eventHubPath, this.partitionId, this.Incarnation);
}
catch (Exception exception)
{
this.host.logger.LogError("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) encountered an exception while processing packets : {exception}", this.host.eventHubPath, this.partitionId, this.Incarnation, exception);
this.partition.ErrorHandler.HandleError("IEventProcessor.ProcessEventsAsync", "Encountered exception while processing events", exception, true, false);
}
this.host.logger.LogInformation("PartitionInstance {eventHubName}/{eventHubPartition}({incarnation}) ReceiverLoop exits", this.host.eventHubPath, this.partitionId, this.Incarnation);
}
