/// <inheritdoc />
public EventHubContentLocationEventStore(
ContentLocationEventStoreConfiguration configuration,
IContentLocationEventHandler eventHandler,
string localMachineName,
CentralStorage centralStorage,
Interfaces.FileSystem.AbsolutePath workingDirectory)
: base(configuration, nameof(EventHubContentLocationEventStore), eventHandler, centralStorage, workingDirectory)
{
Contract.Requires(configuration.MaxEventProcessingConcurrency >= 1);
_configuration = configuration;
_localMachineName = localMachineName;
_eventHubClient = CreateEventHubClient(configuration);
_extraEventHubClientRetryPolicy = CreateEventHubClientRetryPolicy();
if (configuration.MaxEventProcessingConcurrency > 1)
{
_eventProcessingBlocks =
Enumerable.Range(1, configuration.MaxEventProcessingConcurrency)
.Select(
(_, index) =>
{
var serializer = new ContentLocationEventDataSerializer(configuration.SelfCheckSerialization ? ValidationMode.Trace : ValidationMode.Off);
return(new ActionBlock <ProcessEventsInput>(
t => ProcessEventsCoreAsync(t, serializer, index: index),
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1,
BoundedCapacity = configuration.EventProcessingMaxQueueSize,
}));
})
.ToArray();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="EventHubAmqpSink" /> class.
/// </summary>
/// <param name="eventHubConnectionString">The connection string for the eventhub.</param>
/// <param name="eventHubName">The name of the eventhub.</param>
/// <param name="bufferingInterval">The buffering interval between each batch publishing.</param>
/// <param name="bufferingCount">The number of entries that will trigger a batch publishing.</param>
/// <param name="maxBufferSize">The maximum number of entries that can be buffered while it's sending to the store before the sink starts dropping entries.</param>
/// <param name="onCompletedTimeout">Defines a timeout interval for when flushing the entries after an <see cref="OnCompleted"/> call is received and before disposing the sink.
/// This means that if the timeout period elapses, some event entries will be dropped and not sent to the store. Normally, calling <see cref="IDisposable.Dispose"/> on
/// the <see cref="System.Diagnostics.Tracing.EventListener"/> will block until all the entries are flushed or the interval elapses.
/// If <see langword="null"/> is specified, then the call will block indefinitely until the flush operation finishes.</param>
/// <param name="partitionKey">PartitionKey is optional. If no partition key is supplied the log messages are sent to eventhub
/// and distributed to various partitions in a round robin manner.</param>
public EventHubAmqpSink(string eventHubConnectionString, string eventHubName, TimeSpan bufferingInterval, int bufferingCount, int maxBufferSize, TimeSpan onCompletedTimeout, string partitionKey = null)
{
Guard.ArgumentNotNullOrEmpty(eventHubConnectionString, "eventHubConnectionString");
Guard.ArgumentNotNullOrEmpty(eventHubName, "eventHubName");
var factory = MessagingFactory.CreateFromConnectionString(string.Format("{0};TransportType={1}", eventHubConnectionString, TransportType.Amqp));
eventHubClient = new EventHubClientImp(factory.CreateEventHubClient(eventHubName));
SetupSink(bufferingInterval, bufferingCount, maxBufferSize, onCompletedTimeout, partitionKey);
}
private async Task <int> OneThreadAsync(IEventHubClient client, CancellationToken cancellationToken)
{
int eventCount = 0;
while (!cancellationToken.IsCancellationRequested)
{
await client.SendAsync(GetDummyEventObject());
++eventCount;
}
return(eventCount);
}
public void AppendBatchSendMultipleFailureTest()
{
string ConnectionStringName = ConfigurationManager.ConnectionStrings[0].Name;
string EventHubName = "EventHubName";
string connectionString = ConfigurationManager.ConnectionStrings[0].ConnectionString;
string message = "This is a logging message";
Mock <IEventHubClientFactory> mockFactory = new Mock <IEventHubClientFactory>();
Mock <IEventHubClient> mockEventHub = new Mock <IEventHubClient>();
IEventHubClient evenHub = mockEventHub.Object;
mockFactory.Setup(c => c.GetEventHubClient(It.IsAny <string>(), It.IsAny <string>()))
.Callback((string cs, string ehn) => { Assert.AreEqual(connectionString, cs); Assert.AreEqual(EventHubName, ehn); })
.Returns(evenHub);
mockEventHub.SetupSequence(m => m.SendBatchAsync(It.IsAny <IEnumerable <EventData> >()))
.Throws(new Exception("Sending is not working"))
.Throws(new Exception("Sending is not working"))
.Returns(Task.FromResult(0));
mockEventHub.Setup(m => m.CloseAsync()).Returns(Task.FromResult(0));
TestAzureEventHubAppender appender = new TestAzureEventHubAppender(mockFactory.Object)
{
EventHubNamespaceConnectionStringName = ConnectionStringName,
EventHubName = EventHubName
};
PatternLayout patternLayout = new PatternLayout
{
ConversionPattern = "%level:%message"
};
patternLayout.ActivateOptions(); appender.Layout = patternLayout;
appender.MaxRetries = 1;
appender.EventHubRetryDelay = TimeSpan.FromMilliseconds(100);
appender.ActivateOptions();
LoggingEvent loggingEvent = GetLoggingEvent(message);
appender.PublicAppend(loggingEvent);
appender.PublicOnClose();
mockFactory.Verify(mock => mock.GetEventHubClient(It.IsAny <string>(), It.IsAny <string>()), Times.Once);
mockEventHub.Verify(mock => mock.SendBatchAsync(It.IsAny <List <EventData> >()), Times.Exactly(2));
mockEventHub.Verify(mock => mock.CloseAsync(), Times.Once);
}
public void DisposeWithBufferTest()
{
string ConnectionStringName = ConfigurationManager.ConnectionStrings[0].Name;
string EventHubName = "EventHubName";
string connectionString = ConfigurationManager.ConnectionStrings[0].ConnectionString;
string message = "This is a logging message";
Mock <IEventHubClientFactory> mockFactory = new Mock <IEventHubClientFactory>();
Mock <IEventHubClient> mockEventHub = new Mock <IEventHubClient>();
IEventHubClient evenHub = mockEventHub.Object;
mockFactory.Setup(c => c.GetEventHubClient(It.IsAny <string>(), It.IsAny <string>()))
.Callback((string cs, string ehn) => { Assert.AreEqual(connectionString, cs); Assert.AreEqual(EventHubName, ehn); })
.Returns(evenHub);
mockEventHub.Setup(m => m.SendBatchAsync(It.IsAny <IEnumerable <EventData> >()))
.Callback((IEnumerable <EventData> d) => { Assert.AreEqual(1, d.Count()); })
.Returns(Task.FromResult(0));
mockEventHub.Setup(m => m.CloseAsync()).Returns(Task.FromResult(0));
TestAzureEventHubAppender appender = new TestAzureEventHubAppender(mockFactory.Object)
{
EventHubNamespaceConnectionStringName = ConnectionStringName,
EventHubName = EventHubName
};
PatternLayout patternLayout = new PatternLayout
{
ConversionPattern = "%level:%message"
};
patternLayout.ActivateOptions(); appender.Layout = patternLayout;
appender.ActivateOptions();
LoggingEvent loggingEvent = GetLoggingEvent(message);
appender.PublicAppend(loggingEvent);
appender.PublicOnClose();
appender.Dispose();
mockFactory.Verify(mock => mock.GetEventHubClient(It.IsAny <string>(), It.IsAny <string>()), Times.Once);
mockEventHub.Verify(mock => mock.SendBatchAsync(It.IsAny <List <EventData> >()), Times.Once);
mockEventHub.Verify(mock => mock.CloseAsync(), Times.Once);
}
public void AppendMultiEventTest()
{
string ConnectionStringName = ConfigurationManager.ConnectionStrings[0].Name;
string EventHubName = "EventHubName";
string connectionString = ConfigurationManager.ConnectionStrings[0].ConnectionString;
string message = "This is a logging message";
Mock <IEventHubClientFactory> mockFactory = new Mock <IEventHubClientFactory>();
Mock <IEventHubClient> mockEventHub = new Mock <IEventHubClient>();
IEventHubClient evenHub = mockEventHub.Object;
mockFactory.Setup(c => c.GetEventHubClient(It.IsAny <string>(), It.IsAny <string>()))
.Returns(evenHub);
mockEventHub.Setup(m => m.SendBatchAsync(It.IsAny <IEnumerable <EventData> >()))
.Returns(async(IEnumerable <EventData> d) => { await Task.Delay(100); });
mockEventHub.Setup(m => m.CloseAsync()).Returns(Task.FromResult(0));
TestAzureEventHubAppender appender = new TestAzureEventHubAppender(mockFactory.Object)
{
EventHubNamespaceConnectionStringName = ConnectionStringName,
EventHubName = EventHubName
};
PatternLayout patternLayout = new PatternLayout
{
ConversionPattern = "%level:%message"
};
patternLayout.ActivateOptions(); appender.Layout = patternLayout;
appender.ActivateOptions();
LoggingEvent loggingEvent = GetLoggingEvent(message);
appender.PublicAppend(loggingEvent);
appender.PublicAppend(loggingEvent);
appender.PublicAppend(loggingEvent);
appender.PublicOnClose();
mockFactory.Verify(mock => mock.GetEventHubClient(It.IsAny <string>(), It.IsAny <string>()), Times.Once);
mockEventHub.Verify(mock => mock.SendBatchAsync(It.IsAny <List <EventData> >()), Times.Exactly(2));
mockEventHub.Verify(mock => mock.CloseAsync(), Times.Once);
}
public void JsonRenderingTest()
{
string connectionString = config[eventHubConnection];
string message = "This is a logging message";
Mock <IEventHubClientFactory> mockFactory = new Mock <IEventHubClientFactory>();
Mock <IEventHubClient> mockEventHub = new Mock <IEventHubClient>();
IEventHubClient evenHub = mockEventHub.Object;
mockFactory.Setup(c => c.GetEventHubClient(It.IsAny <string>()))
.Callback((string cs) => { Assert.AreEqual(connectionString, cs); })
.Returns(evenHub);
mockEventHub.SetupSequence(m => m.SendAsync(It.IsAny <IEnumerable <EventData> >()))
.Throws(new Exception("Sending is not working"))
.Returns(Task.FromResult(0));
mockEventHub.Setup(m => m.CloseAsync()).Returns(Task.FromResult(0));
TestAzureEventHubAppender appender = new TestAzureEventHubAppender(mockFactory.Object)
{
ConnectionString = connectionString
};
PatternLayout patternLayout = new PatternLayout
{
ConversionPattern = "level:%level;message:%message"
};
patternLayout.ActivateOptions(); appender.Layout = patternLayout;
appender.ActivateOptions();
LoggingEvent loggingEvent = GetLoggingEvent(message);
appender.PublicAppend(loggingEvent);
appender.PublicOnClose();
mockFactory.Verify(mock => mock.GetEventHubClient(It.IsAny <string>()), Times.Once);
mockEventHub.Verify(mock => mock.SendAsync(It.IsAny <List <EventData> >()), Times.Exactly(2));
mockEventHub.Verify(mock => mock.CloseAsync(), Times.Once);
}
public void AppendGracefulExitTest()
{
string connectionString = config[eventHubConnection];
string message = "This is a logging message";
Mock <IEventHubClientFactory> mockFactory = new Mock <IEventHubClientFactory>();
Mock <IEventHubClient> mockEventHub = new Mock <IEventHubClient>();
IEventHubClient evenHub = mockEventHub.Object;
mockFactory.Setup(c => c.GetEventHubClient(It.IsAny <string>()))
.Returns(evenHub);
mockEventHub.Setup(m => m.SendAsync(It.IsAny <IEnumerable <EventData> >()))
.Returns(async(IEnumerable <EventData> d) => { await Task.Delay(100); });
mockEventHub.Setup(m => m.CloseAsync()).Returns(Task.FromResult(0));
TestAzureEventHubAppender appender = new TestAzureEventHubAppender(mockFactory.Object)
{
ConnectionString = connectionString
};
PatternLayout patternLayout = new PatternLayout
{
ConversionPattern = "%level:%message"
};
patternLayout.ActivateOptions(); appender.Layout = patternLayout;
appender.ActivateOptions();
LoggingEvent loggingEvent = GetLoggingEvent(message);
appender.PublicAppend(loggingEvent);
Thread.Sleep(500); // Let the sender send the event - so that the call to Take will hit the InvalidOperationException
appender.PublicOnClose();
mockFactory.Verify(mock => mock.GetEventHubClient(It.IsAny <string>()), Times.Once);
mockEventHub.Verify(mock => mock.SendAsync(It.IsAny <List <EventData> >()), Times.Once);
mockEventHub.Verify(mock => mock.CloseAsync(), Times.Once);
}
public void EventHubFailsInitializationTest()
{
string ConnectionStringName = ConfigurationManager.ConnectionStrings[0].Name;
string EventHubName = "EventHubName";
string connectionString = ConfigurationManager.ConnectionStrings[0].ConnectionString;
Mock <IEventHubClientFactory> mockFactory = new Mock <IEventHubClientFactory>();
Mock <IEventHubClient> mockEventHub = new Mock <IEventHubClient>();
IEventHubClient evenHub = mockEventHub.Object;
mockFactory.Setup(c => c.GetEventHubClient(It.IsAny <string>(), It.IsAny <string>()))
.Throws(new Exception("Failed"));
mockEventHub.Setup(m => m.SendBatchAsync(It.IsAny <IEnumerable <EventData> >()))
.Callback((IEnumerable <EventData> d) => { Assert.AreEqual(1, d.Count()); })
.Returns(Task.FromResult(0));
mockEventHub.Setup(m => m.CloseAsync()).Returns(Task.FromResult(0));
TestAzureEventHubAppender appender = new TestAzureEventHubAppender(mockFactory.Object)
{
EventHubNamespaceConnectionStringName = ConnectionStringName,
EventHubName = EventHubName
};
PatternLayout patternLayout = new PatternLayout
{
ConversionPattern = "%level:%message"
};
patternLayout.ActivateOptions(); appender.Layout = patternLayout;
appender.ActivateOptions();
appender.PublicOnClose();
mockFactory.Verify(mock => mock.GetEventHubClient(It.IsAny <string>(), It.IsAny <string>()), Times.Once);
mockEventHub.Verify(mock => mock.CloseAsync(), Times.Never);
}
/// <summary>
/// Sending task - this reads EventData from the buffer - gethers up a batch (if avaialable) and send to the event hub
/// </summary>
/// <returns></returns>
private async Task Sender()
{
var connectionString = GetConfiguredConnectionString(EventHubNamespaceConnectionStringName);
var eventHubName = GetConfiguredEventHubName();
try
{
// Open session with the Event Hub.
//
IEventHubClient eventHubClient = EventHubClientFactory.GetEventHubClient(connectionString, eventHubName);
while (!Buffer.IsCompleted) // run while there is data avaialble
{
EventData item = null;
try
{
item = Buffer.Take(); // Get the first available item - this will block until there is data
}
catch (InvalidOperationException)
{
continue; // Nothing left to do - this task is done - clean up and leave
}
int additional = GetAdditionalItemCount(Buffer.Count);
// Build the batch
List <EventData> batch = new List <EventData> {
item
};
while (additional > 0)
{
batch.Add(Buffer.Take());
additional--;
}
bool batchSent = false;
int attempt = 0;
while (!batchSent)
{
try
{
attempt++;
// Send the batch
await eventHubClient.SendBatchAsync(batch).ConfigureAwait(false);
batchSent = true;
}
catch (Exception exception)
{
string message = $"Exception while sending Batch to EventHub({EventHubName})";
Trace.TraceError(message + " - " + exception.ToString());
ErrorHandler.Error(message, exception);
if (attempt > MaxRetries)
{
Trace.TraceError($"Tried {attempt} time to send a batch to Event Hub({EventHubName}) - giving up");
ErrorHandler.Error($"Tried {attempt} time to send a batch to Event Hub({EventHubName}) - giving up");
batchSent = true;
}
else
{
await Task.Delay(EventHubRetryDelay).ConfigureAwait(false);
}
}
}
// Not really an error - but debugging information so see how things are working (or not).
//
ErrorHandler.Error($"Sent batch of size {batch.Count}");
}
await eventHubClient.CloseAsync().ConfigureAwait(false);
}
catch (Exception exception)
{
string message = $"There was a problem with the EventHub({EventHubName})";
Trace.TraceError(message);
ErrorHandler.Error(message, exception);
}
}
public AzureEventHubCommandAuditor(IEventHubClient client, IEventHubSerializer serializer, IPartitionKeyProvider partitionKeyProvider)
{
_client = client;
_serializer = serializer;
_partitionKeyProvider = partitionKeyProvider;
}
public void ReleaseClient(IEventHubClient client)
{
_queue.TryAdd(client);
}
public Need(IEventMessage eventMessage, IEventHubClient eventHubClient)
{
_eventMessage = eventMessage;
_eventHubClient = eventHubClient;
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
// Get a reference to the current connections array first, just in case there is another thread wanting to clean
// up the connections with CleanUpAsync(), we won't get a null reference exception here.
IEventHubClient[] currentClients = Interlocked.CompareExchange <IEventHubClient[]>(ref this.clients, this.clients, this.clients);
if (currentClients == null || events == null || events.Count == 0)
{
return;
}
try
{
var groupedEventData = events.GroupBy(e => string.IsNullOrEmpty(outputConfiguration.PartitionKeyProperty) == false && e.TryGetPropertyValue(outputConfiguration.PartitionKeyProperty, out var partionKeyData) ? partionKeyData.ToString() : string.Empty, e => e);
List <Task> tasks = new List <Task>();
foreach (var partitionedEventData in groupedEventData)
{
//assemble the full list of MessagingEventData items plus their messageSize
List <(MessagingEventData message, int messageSize)> batchRecords = partitionedEventData.Select(
e => new Tuple <MessagingEventData, int>(e.ToMessagingEventData(SerializerSettings, out var messageSize), messageSize).ToValueTuple()).ToList();
SendBatch(batchRecords);
void SendBatch(IReadOnlyCollection <(MessagingEventData message, int messageSize)> batch)
{
// Since event hub limits each message/batch to be a certain size, we need to
// keep checking the size in bytes of the batch and recursively keep splitting into two batches as needed
if (batch.Count >= 2 && batch.Sum(b => b.messageSize) > EventHubMessageSizeLimit)
{
//the batch total message size is too big to send to EventHub, but it still contains at least two items,
//so we split the batch up in half and recusively call the inline SendBatch() method with the two new smaller batches
var indexMiddle = batch.Count / 2;
SendBatch(batch.Take(indexMiddle).ToList());
SendBatch(batch.Skip(indexMiddle).ToList());
return;
}
IEventHubClient hubClient = currentClients[transmissionSequenceNumber % ConcurrentConnections];
if (string.IsNullOrEmpty(partitionedEventData.Key))
{
tasks.Add(hubClient.SendAsync(batch.Select(b => b.message)));
}
else
{
tasks.Add(hubClient.SendAsync(batch.Select(b => b.message), partitionedEventData.Key));
}
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
}
await Task.WhenAll(tasks).ConfigureAwait(false);
this.healthReporter.ReportHealthy();
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = nameof(EventHubOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
public AzureEventHubDispatcher(IEventHubClient client, Func <ICommand, string> getPartitionKeyFunc = null)
{
_client = client;
_getPartitionKeyFunc = getPartitionKeyFunc;
}
internal EventHubAmqpSink(IEventHubClient eventHubClient, TimeSpan bufferingInterval, int bufferingCount, int maxBufferSize, TimeSpan onCompletedTimeout, string partitionKey = null)
{
this.eventHubClient = eventHubClient;
SetupSink(bufferingInterval, bufferingCount, maxBufferSize, onCompletedTimeout, partitionKey);
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
// Get a reference to the current connections array first, just in case there is another thread wanting to clean
// up the connections with CleanUpAsync(), we won't get a null reference exception here.
IEventHubClient[] currentClients = Interlocked.CompareExchange <IEventHubClient[]>(ref this.clients, this.clients, this.clients);
if (currentClients == null || events == null || events.Count == 0)
{
return;
}
try
{
// Since event hub limits each message/batch to be a certain size, we need to
// keep checking the size for exceeds and split into a new batch as needed
List <List <MessagingEventData> > batches = new List <List <MessagingEventData> >();
int batchByteSize = 0;
foreach (EventData eventData in events)
{
int messageSize;
MessagingEventData messagingEventData = eventData.ToMessagingEventData(out messageSize);
// If we don't have a batch yet, or the addition of this message will exceed the limit for this batch, then
// start a new batch.
if (batches.Count == 0 ||
batchByteSize + messageSize > EventHubMessageSizeLimit)
{
batches.Add(new List <MessagingEventData>());
batchByteSize = 0;
}
batchByteSize += messageSize;
List <MessagingEventData> currentBatch = batches[batches.Count - 1];
currentBatch.Add(messagingEventData);
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
IEventHubClient hubClient = currentClients[transmissionSequenceNumber % ConcurrentConnections];
List <Task> tasks = new List <Task>();
foreach (List <MessagingEventData> batch in batches)
{
tasks.Add(hubClient.SendAsync(batch));
}
await Task.WhenAll(tasks).ConfigureAwait(false);
this.healthReporter.ReportHealthy();
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = nameof(EventHubOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
private async Task Sender()
{
var connectionString = this.ConnectionString;
var eventHubName = this.EventHubName;
try
{
// Open session with the Event Hub.
if (this.eventHubClient == null && !string.IsNullOrEmpty(this.ConnectionString))
{
this.eventHubClient = EventHubClientFactory.GetEventHubClient(this.ConnectionString);
}
while (!Buffer.IsCompleted) // run while there is data avaialble
{
EventData item = null;
try
{
item = Buffer.Take(); // Get the first available item - this will block until there is data
}
catch (InvalidOperationException)
{
continue; // Nothing left to do - this task is done - clean up and leave
}
int additional = GetAdditionalItemCount(Buffer.Count);
// Build the batch
List <EventData> batch = new List <EventData> {
item
};
while (additional > 0)
{
batch.Add(Buffer.Take());
additional--;
}
bool batchSent = false;
int attempt = 0;
while (!batchSent)
{
try
{
attempt++;
// Send the batch
await eventHubClient.SendAsync(batch).ConfigureAwait(false);
batchSent = true;
}
catch (Exception exception)
{
string message = $"Exception while sending Batch to EventHub({EventHubName})";
Trace.TraceError(message + " - " + exception.ToString());
ErrorHandler.Error(message, exception);
//If the connection string is wrong, we'll never get the messages through, so stop trying (covers bad host name, bad eventhub name (status 404) and bad signature (status 401)
if (exception.Message.ToLower().Contains("no such host") || exception.Message.ToLower().Contains("status-code: 404") || exception.Message.ToLower().Contains("status-code: 401"))
{
return;
}
if (attempt > MaxRetries)
{
Trace.TraceError($"Tried {attempt} time to send a batch to Event Hub({EventHubName}) - giving up");
ErrorHandler.Error($"Tried {attempt} time to send a batch to Event Hub({EventHubName}) - giving up");
batchSent = true;
}
else
{
await Task.Delay(EventHubRetryDelay).ConfigureAwait(false);
}
}
}
// Not really an error - but debugging information so see how things are working (or not).
//
ErrorHandler.Error($"Sent batch of size {batch.Count}");
}
await eventHubClient.CloseAsync().ConfigureAwait(false);
}
catch (Exception exception)
{
string message = $"There was a problem with the EventHub({EventHubName})";
Trace.TraceError(message);
ErrorHandler.Error(message, exception);
}
}
