private static int Add <T>(ref EventDataBatch batch, T msg)
{
int retval = 0;
var data = new EventData(BinarySerializer.Serialize(msg));
data.Properties.Add("Table", typeof(T).Name);
data.Properties.Add("Format", "json");
data.Properties.Add("IngestionMappingReference", $"Map{typeof(T).Name}");
//var back = BinarySerializer.DeserializeStreamEventSampleEvent(data);
if (!batch.TryAdd(data))
{
Console.WriteLine($"Sending #{batch.Count} messages");
retval = batch.Count;
eventHubClient.SendAsync(batch).Wait();
batch = eventHubClient.CreateBatch(new BatchOptions
{
MaxMessageSize = 1046528,
PartitionKey = "default"
});
if (!batch.TryAdd(data))
{
throw new OverflowException("batch size to low");
}
}
return(retval);
}
private void TransformToBatch()
{
EventDataBatch batch = _client.CreateBatch();
bool stopCollectingEvents = false;
if (_next != null && batch.TryAdd(_next))
{
_next = null;
}
while (!stopCollectingEvents)
{
if (_input.TryDequeue(out EventData data))
{
if (!batch.TryAdd(data))
{
batch = _client.CreateBatch();
_next = data;
}
}
stopCollectingEvents = _next != null || data == null;
}
_output.Enqueue(batch.ToEnumerable());
}
/// <inheritdoc/>
public async Task PublishAsync(IEnumerable <Message> messages, CancellationToken cancellationToken = default)
{
_ = messages ?? throw new ArgumentNullException(nameof(messages));
if (messages.Any(m => m.EnqueueTime.HasValue))
{
throw new ArgumentException("The EnqueueTime property is not supported with Event Hubs.");
}
try
{
await Task.WhenAll(messages
.GroupBy(m => m.PartitionId)
.Select(messageGroup => Task.Run(async() =>
{
var batch = _client.CreateBatch(new BatchOptions
{
PartitionKey = messageGroup.Key,
MaxMessageSize = _maxMessageSize
});
try
{
foreach (var message in messageGroup)
{
if (!batch.TryAdd(CreateEventData(message)))
{
await _client.SendAsync(batch);
batch.Dispose();
batch = _client.CreateBatch(new BatchOptions
{
PartitionKey = messageGroup.Key,
MaxMessageSize = _maxMessageSize
});
}
}
if (batch.Count > 0)
{
await _client.SendAsync(batch);
}
}
finally
{
batch.Dispose();
}
}))).ConfigureAwait(false);
}
catch (TaskCanceledException) { throw; }
catch (Exception e)
{
throw new MessagePublishingFailedException(null, "Could not publish some of the messages.", e);
}
}
public async Task PublishBatchEventsAsync <T>(IEnumerable <T> events) where T : VacancyEvent
{
if (events.Count() < 1)
{
throw new ArgumentException("Must supply at least one event to publish as part of batch");
}
var exMsg = $"Error publishing batch events: {typeof(T).Name}.";
try
{
var batchedEvents = _client.CreateBatch();
var addedSuccesfully = PopulateBatch(events, batchedEvents);
if (!addedSuccesfully)
{
throw new BatchEventPublishException(exMsg);
}
await _client.SendAsync(batchedEvents);
await _client.CloseAsync();
}
catch (EventHubsException ex)
{
_logger.LogError(exMsg, ex);
}
}
public async Task SendDataAsync(IEnumerable <byte[]> datas)
{
var datasToHub = datas.Select(eventbytes => new EventData(eventbytes));
var eventHubBatch = _eventHubClient.CreateBatch();
Parallel.ForEach(datasToHub, async(dataToHub) => {
if (!eventHubBatch.TryAdd(dataToHub))
{
await _eventHubClient.SendAsync(eventHubBatch.ToEnumerable());
eventHubBatch = _eventHubClient.CreateBatch();
eventHubBatch.TryAdd(dataToHub);
}
});
if (eventHubBatch.Count > 0)
{
await _eventHubClient.SendAsync(datasToHub);
}
}
private async Task PushBatchToEventHubAsync(IList <AsyncLogEventInfo> logEvents)
{
var batch = _client.CreateBatch();
foreach (var logEvent in logEvents)
{
batch.TryAdd(new EventData(Encoding.UTF8.GetBytes(logEvent.LogEvent.FormattedMessage)));
}
await _client.SendAsync(batch);
}
public async Task SendDataAsync(IEnumerable <CoffeeMachineData> datas)
{
var eventDatas = datas.Select(x => CreateEventData(x));
var evDataBatch = _ehClient.CreateBatch();
foreach (var evData in eventDatas)
{
if (!evDataBatch.TryAdd(evData))
{
await _ehClient.SendAsync(evDataBatch);
evDataBatch = _ehClient.CreateBatch();
evDataBatch.TryAdd(evData);
}
}
if (evDataBatch.Count > 0)
{
await _ehClient.SendAsync(evDataBatch);
}
}
public async Task SendAsync(IEnumerable <EventMessageModel> messages)
{
var eventDataBatch = _eventHubClient.CreateBatch();
var eventData = messages.ToEventData();
foreach (var data in eventData)
{
if (!eventDataBatch.TryAdd(data))
{
await SendBatchAsync(eventDataBatch);
eventDataBatch = _eventHubClient.CreateBatch();
eventDataBatch.TryAdd(data);
}
}
if (eventDataBatch.Count > 0)
{
await SendBatchAsync(eventDataBatch);
}
}
/// <summary>
/// Send data to event hubs defined in appconfig
/// </summary>
/// <param name="ehConnectionString"></param>
/// <param name="data"></param>
/// <returns></returns>
public async Task SendToEventHubBatch(string ehConnectionString, List <JObject> data)
{
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(ehConnectionString);
var batchevents = eventHubClient.CreateBatch();
foreach (var deviceData in data)
{
var eventData = new EventData(Encoding.ASCII.GetBytes(deviceData.ToString(Formatting.None)));
if (!batchevents.TryAdd(eventData))
{
await eventHubClient.SendAsync(batchevents);
batchevents = eventHubClient.CreateBatch();
batchevents.TryAdd(eventData);
}
}
if (batchevents.Count > 0)
{
await eventHubClient.SendAsync(batchevents);
}
await eventHubClient.CloseAsync();
}
public async Task SendDataAsync(IEnumerable <CoffeeMachineData> coffeeMachineData)
{
var eventsData = coffeeMachineData.Select(e => CreateEventData(e));
EventDataBatch eventDataBatch = _eventHubClient.CreateBatch();
foreach (EventData eventData in eventsData)
{
if (!eventDataBatch.TryAdd(eventData))
{
await _eventHubClient.SendAsync(eventDataBatch.ToEnumerable());
eventDataBatch = _eventHubClient.CreateBatch();
eventDataBatch.TryAdd(eventData);
}
}
if (eventDataBatch.Count > 0)
{
await _eventHubClient.SendAsync(eventDataBatch);
}
}
public async Task SendDataAsync(IEnumerable <SweetsMachineData> sweetsMachineEvents)
{
var eventDatas = sweetsMachineEvents.Select(sweetsMachineEvent => CreateEventData(sweetsMachineEvent));
var eventDataBatch = _eventHubClient.CreateBatch();
foreach (var eventData in eventDatas)
{
if (!eventDataBatch.TryAdd(eventData))
{
await _eventHubClient.SendAsync(eventDataBatch);
eventDataBatch = _eventHubClient.CreateBatch();
eventDataBatch.TryAdd(eventData);
}
}
if (eventDataBatch.Count > 0)
{
await _eventHubClient.SendAsync(eventDataBatch);
}
}
// EventHubClient comes with an overloaded version of SendAsync that accepts IEnumerable
public async Task SendDataAsync(IEnumerable <CoffeeMachineData> datas)
{
var eventDatas = datas.Select(coffeeMachineData => CreateEventData(coffeeMachineData));
// Creates an eventhub batch that prevents batches from exceeding the size limit of 256kb
var eventDataBatch = _eventHubClient.CreateBatch();
foreach (var eventData in eventDatas)
{
// handles if the event date would exceed the limit
if (!eventDataBatch.TryAdd(eventData))
{
await _eventHubClient.SendAsync(eventDataBatch.ToEnumerable());
eventDataBatch = _eventHubClient.CreateBatch();
eventDataBatch.TryAdd(eventData);
}
}
if (eventDataBatch.Count > 0)
{
await _eventHubClient.SendAsync(eventDataBatch.ToEnumerable());
}
}
public async Task SendDataAsync(IEnumerable <TerminalData> datas)
{
var eventDatas = datas.Select(terminalData => CreateEventData(terminalData));
//CreateBatch() ya tiene definido el tamaño correcto para el mensaje
var eventDataBatch = _eventHubClient.CreateBatch();
foreach (var eventData in eventDatas)
{
if (!eventDataBatch.TryAdd(eventData))
{
await _eventHubClient.SendAsync(eventDataBatch);
eventDataBatch = _eventHubClient.CreateBatch();
eventDataBatch.TryAdd(eventData);
}
}
if (eventDataBatch.Count > 0)
{
await _eventHubClient.SendAsync(eventDataBatch);
}
}
public async Task Send(IEnumerable <Message> messages, string partitionKey)
{
var options = new BatchOptions {
PartitionKey = partitionKey
};
EventDataBatch batch = _eventHub.CreateBatch(options);
foreach (EventData eventData in messages.Select(_converter.ConvertToEvent))
{
if (batch.TryAdd(eventData) == false)
{
await _eventHub.SendAsync(batch).ConfigureAwait(continueOnCapturedContext: false);
batch = _eventHub.CreateBatch(options);
batch.TryAdd(eventData);
}
}
if (batch.Count > 0)
{
await _eventHub.SendAsync(batch).ConfigureAwait(continueOnCapturedContext: false);
}
}
public async Task SendDataAsync(IEnumerable <CoffeeMachineData> coffeeMachineDatas)
{
var eventDatas = coffeeMachineDatas.Select(coffeeMachineData => CreateEventData(coffeeMachineData));
var eventDataBatch = _eventHubClient.CreateBatch();
foreach (var eventData in eventDatas)
{
if (!eventDataBatch.TryAdd(eventData))
{
//if batch is full send what is currently batched and then create a new batch and add
//the eventData object that could not be added to the batch that was sent
await _eventHubClient.SendAsync(eventDataBatch.ToEnumerable());
eventDataBatch = _eventHubClient.CreateBatch();
eventDataBatch.TryAdd(eventData);
}
}
ole
if (eventDataBatch.Count > 0)
{
await _eventHubClient.SendAsync(eventDataBatch.ToEnumerable());
}
}
/// <summary>
/// Executes the performance test scenario asynchronously.
/// </summary>
///
/// <param name="cancellationToken">The token used to signal when cancellation is requested.</param>
///
public async override Task RunAsync(CancellationToken cancellationToken)
{
using var batch = s_client.CreateBatch();
// Fill the batch with events using the same body. This will result in a batch of events of equal size.
// The events will only differ by the id property that is assigned to them.
foreach (var eventData in EventGenerator.CreateEventsFromBody(Options.Count, s_eventBody))
{
if (!batch.TryAdd(eventData))
{
throw new InvalidOperationException("It was not possible to fit the requested number of events in a single batch.");
}
}
await s_client.SendAsync(batch).ConfigureAwait(false);
}
public async Task PublishToSpecificPartition()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Migrate_T1_PublishToSpecificPartition
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = TestUtility.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var builder = new EventHubsConnectionStringBuilder(connectionString);
builder.EntityPath = eventHubName;
EventHubClient client = EventHubClient.CreateFromConnectionString(builder.ToString());
PartitionSender sender = default;
try
{
using var eventBatch = client.CreateBatch();
for (var index = 0; index < 5; ++index)
{
var eventData = new EventData(Encoding.UTF8.GetBytes($"Event #{ index }"));
if (!eventBatch.TryAdd(eventData))
{
throw new Exception($"The event at { index } could not be added");
}
}
string firstPartition = (await client.GetRuntimeInformationAsync()).PartitionIds.First();
sender = client.CreatePartitionSender(firstPartition);
await sender.SendAsync(eventBatch);
}
finally
{
sender?.Close();
client.Close();
}
#endregion
}
public async Task PublishWithAPartitionKey()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Migrate_T1_PublishWithAPartitionKey
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = TestUtility.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var builder = new EventHubsConnectionStringBuilder(connectionString);
builder.EntityPath = eventHubName;
EventHubClient client = EventHubClient.CreateFromConnectionString(builder.ToString());
try
{
var batchOptions = new BatchOptions
{
PartitionKey = "Any Value Will Do..."
};
using var eventBatch = client.CreateBatch(batchOptions);
for (var index = 0; index < 5; ++index)
{
var eventData = new EventData(Encoding.UTF8.GetBytes($"Event #{ index }"));
if (!eventBatch.TryAdd(eventData))
{
throw new Exception($"The event at { index } could not be added");
}
}
await client.SendAsync(eventBatch);
}
finally
{
client.Close();
}
#endregion
}
/// <summary>
/// Performs the tasks needed to initialize and set up the environment for the test scenario.
/// When multiple instances are run in parallel, the setup will take place once, prior to the
/// execution of the first test instance.
/// </summary>
///
public async override Task GlobalSetupAsync()
{
await base.GlobalSetupAsync().ConfigureAwait(false);
s_scope = await EventHubScope.CreateAsync(4).ConfigureAwait(false);
s_client = EventHubClient.CreateFromConnectionString(TestUtility.BuildEventHubsConnectionString(s_scope.EventHubName));
s_eventBody = EventGenerator.CreateRandomBody(Options.Size);
// Publish an empty event to force the connection and link to be established.
using var batch = s_client.CreateBatch();
if (!batch.TryAdd(new EventData(Array.Empty <byte>())))
{
throw new InvalidOperationException("The empty event could not be added to the batch during global setup.");
}
await s_client.SendAsync(batch).ConfigureAwait(false);
}
/// <summary>
/// Publish events in batch
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="myEvents"></param>
public async Task PublishInBatch <T>(IEnumerable <T> myEvents)
{
try
{
if (myEvents == null)
{
throw new ArgumentNullException(nameof(myEvents));
}
var batchData = _eventHubClient.CreateBatch();
foreach (var myEvent in myEvents)
{
// 1. Serialize the event
var serializedEvent = JsonConvert.SerializeObject(myEvent);
// 2. Convert serialized event to byte array
var eventBytes = Encoding.UTF8.GetBytes(serializedEvent);
// 3. Add event in Batch object
var res = batchData.TryAdd(new EventData(eventBytes));
if (!res)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Error!");
Console.WriteLine("Batch's size limit exceeded. It will be send the previous events added");
break;
}
}
// 4. Publish the events
await _eventHubClient.SendAsync(batchData);
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Error!");
Console.WriteLine(ex);
}
}
/// <summary>
/// Builds a set of batches from the provided events.
/// </summary>
///
/// <param name="events">The events to group into batches.</param>
/// <param name="client">The client to use for creating batches.</param>
/// <param name="batchEvents">A dictionary to which the events included in the batches should be tracked.</param>
/// <param name="batchOptions">The set of options to apply when creating batches.</param>
///
/// <returns>The set of batches needed to contain the entire set of <paramref name="events"/>.</returns>
///
/// <remarks>
/// Callers are assumed to be responsible for taking ownership of the lifespan of the returned batches, including
/// their disposal.
///
/// This method is intended for use within the test suite only; it is not hardened for general purpose use.
/// </remarks>
///
public static IEnumerable <EventDataBatch> BuildBatchesAsync(IEnumerable <EventData> events,
EventHubClient client,
BatchOptions batchOptions = default)
{
EventData eventData;
var queuedEvents = new Queue <EventData>(events);
var batches = new List <EventDataBatch>();
var currentBatch = default(EventDataBatch);
while (queuedEvents.Count > 0)
{
currentBatch ??= client.CreateBatch(batchOptions);
eventData = queuedEvents.Peek();
if (!currentBatch.TryAdd(eventData))
{
if (currentBatch.Count == 0)
{
throw new InvalidOperationException("There was an event too large to fit into a batch.");
}
batches.Add(currentBatch);
currentBatch = default;
}
else
{
queuedEvents.Dequeue();
}
}
if ((currentBatch != default) && (currentBatch.Count > 0))
{
batches.Add(currentBatch);
}
return(batches);
}
public static async Task Run(
[EventHubTrigger("%SourceEventHubName%", Connection = "EventHubConnectionString")] EventData[] messages,
[EventHub("%DestinationEventHubName%", Connection = "EventHubConnectionString")] EventHubClient archiveEventHubClient,
[ServiceBus("%ChatTopicPath%", Connection = "ServiceBusConnectionString")] MessageSender topicClient,
ILogger log)
{
log.LogInformation("EventHubTrigger was called.");
InitEnvironmentVars();
if (topicClient == null || topicClient.IsClosedOrClosing)
{
log.LogInformation("topicClient == null || topicClient.IsClosedOrClosing");
topicClient = new MessageSender("ServiceBusConnectionString", _chatTopic);
}
// Reuse the HttpClient across calls as much as possible so as not to exhaust all available sockets on the server on which it runs.
_sentimentClient = _sentimentClient ?? new HttpClient();
_intentClient = _intentClient ?? new HttpClient();
_sentimentClient.DefaultRequestHeaders.Clear();
_sentimentClient.DefaultRequestHeaders.Accept.Clear();
_sentimentClient.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", _textAnalyticsAccountKey);
_sentimentClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
foreach (var eventData in messages)
{
try
{
//TODO: 1.Extract the JSON payload from the binary message
string sourceEventHubEventBody = Encoding.UTF8.GetString(eventData.Body);
var sentimentMessage = JsonConvert.DeserializeObject <MessageType>(sourceEventHubEventBody);
//TODO: 6 Append sentiment score to chat message object
if (sentimentMessage.messageType.Equals("chat", StringComparison.OrdinalIgnoreCase))
{
sentimentMessage.score = await GetSentimentScore(sentimentMessage);
log.LogInformation("SentimentScore: " + sentimentMessage.score);
}
//TODO: 2.Create a Message (for Service Bus) and EventData instance (for EventHubs) from source message body
var updatedMessage = JsonConvert.SerializeObject(sentimentMessage);
var chatMessage = new Message(Encoding.UTF8.GetBytes(updatedMessage));
// Write the body of the message to the console.
log.LogInformation($"Sending message: {updatedMessage}");
//TODO: 3.Copy the message properties from source to the outgoing message instances
foreach (var prop in eventData.Properties)
{
chatMessage.UserProperties.Add(prop.Key, prop.Value);
}
//TODO: 4.Send chat message to Topic
//Send the message to the topic which will be eventually picked up by ChatHub.cs in the web app.
await topicClient.SendAsync(chatMessage);
//TODO: 5.Send chat message to next EventHub (for archival)
using var eventBatch = archiveEventHubClient.CreateBatch();
EventData updatedEventData = new EventData(Encoding.UTF8.GetBytes(updatedMessage));
eventBatch.TryAdd(updatedEventData);
await archiveEventHubClient.SendAsync(eventBatch);
log.LogInformation("Forwarded message to event hub.");
//TODO: 7.Respond to chat message intent if appropriate
var updatedMessageObject = JsonConvert.DeserializeObject <MessageType>(updatedMessage);
// Get your most likely intent based on your message.
var intent = await GetIntentAndEntities(updatedMessageObject.message);
await HandleIntent(intent, updatedMessageObject, topicClient);
}
catch (Exception ex)
{
// Need to include the stack trace so you can see the specific details of where the error is occuring.
log.LogError(ex.StackTrace);
log.LogError("Chat message processor encountered error while processing", ex);
// Need to rethrow the exception so the Azure Function logs show a failure.
throw ex;
}
}
}