// The Initialize method is not thread-safe. Please only call this on one thread and do so before the pipeline starts sending
// data to this output
private void Initialize(EventHubOutputConfiguration configuration)
{
Debug.Assert(configuration != null);
Debug.Assert(this.healthReporter != null);
if (string.IsNullOrWhiteSpace(configuration.ConnectionString))
{
var errorMessage = $"{nameof(EventHubOutput)}: '{nameof(EventHubOutputConfiguration.ConnectionString)}' configuration parameter must be set to a valid connection string";
healthReporter.ReportProblem(errorMessage, EventFlowContextIdentifiers.Configuration);
throw new Exception(errorMessage);
}
EventHubsConnectionStringBuilder connStringBuilder = new EventHubsConnectionStringBuilder(configuration.ConnectionString);
this.eventHubName = connStringBuilder.EntityPath ?? configuration.EventHubName;
if (string.IsNullOrWhiteSpace(this.eventHubName))
{
var errorMessage = $"{nameof(EventHubOutput)}: Event Hub name must not be empty. It can be specified in the '{nameof(EventHubOutputConfiguration.ConnectionString)}' or '{nameof(EventHubOutputConfiguration.EventHubName)}' configuration parameter";
healthReporter.ReportProblem(errorMessage, EventFlowContextIdentifiers.Configuration);
throw new Exception(errorMessage);
}
connStringBuilder.EntityPath = this.eventHubName;
this.clients = new EventHubClient[ConcurrentConnections];
for (uint i = 0; i < this.clients.Length; i++)
{
this.clients[i] = EventHubClient.CreateFromConnectionString(connStringBuilder.ToString());
}
}
public EventHubOutput(EventHubOutputConfiguration eventHubOutputConfiguration, IHealthReporter healthReporter)
{
Requires.NotNull(eventHubOutputConfiguration, nameof(eventHubOutputConfiguration));
Requires.NotNull(healthReporter, nameof(healthReporter));
this.healthReporter = healthReporter;
Initialize(eventHubOutputConfiguration);
}
public EventHubOutput(
EventHubOutputConfiguration eventHubOutputConfiguration,
IHealthReporter healthReporter,
Func <string, IEventHubClient> eventHubClientFactory = null)
{
Requires.NotNull(eventHubOutputConfiguration, nameof(eventHubOutputConfiguration));
Requires.NotNull(healthReporter, nameof(healthReporter));
this.healthReporter = healthReporter;
this.eventHubClientFactory = eventHubClientFactory ?? this.CreateEventHubClient;
this.outputConfiguration = eventHubOutputConfiguration;
Initialize();
}
private IEventHubClient CreateEventHubClient(EventHubOutputConfiguration _)
{
Debug.Assert(this.outputConfiguration != null);
if (this.outputConfiguration.UseAzureIdentity)
{
this.eventHubName = this.outputConfiguration.EventHubName;
ensureEventHubName();
if (string.IsNullOrWhiteSpace(this.outputConfiguration.FullyQualifiedNamespace))
{
var emptyNamespaceMsg = $"{nameof(EventHubOutput)}: Event Hub namespace must not be empty when using Azure Identity. It can be specified in the '{nameof(EventHubOutputConfiguration.FullyQualifiedNamespace)}' configuration parameter";
healthReporter.ReportProblem(emptyNamespaceMsg, EventFlowContextIdentifiers.Configuration);
throw new Exception(emptyNamespaceMsg);
}
TokenCredential azureTokenCredential = this.outputConfiguration.AzureTokenCredential ?? new DefaultAzureCredential();
return(new EventHubClientImpl(
new EventHubProducerClient(this.outputConfiguration.FullyQualifiedNamespace, this.eventHubName, azureTokenCredential)
));
}
if (!string.IsNullOrWhiteSpace(this.outputConfiguration.ConnectionString))
{
var connString = EventHubsConnectionStringProperties.Parse(this.outputConfiguration.ConnectionString);
this.eventHubName = connString.EventHubName ?? this.outputConfiguration.EventHubName;
ensureEventHubName();
return(new EventHubClientImpl(
new EventHubProducerClient(this.outputConfiguration.ConnectionString, this.eventHubName)
));
}
var invalidConfigMsg =
$"Invalid {nameof(EventHubOutput)} configuration encountered: '{nameof(EventHubOutputConfiguration.ConnectionString)}' value is empty and '{nameof(EventHubOutputConfiguration.UseAzureIdentity)}' is set to false. " +
$"You need to specify either '{nameof(EventHubOutputConfiguration.ConnectionString)}' to EventHub or set '{nameof(EventHubOutputConfiguration.UseAzureIdentity)}' flag.";
healthReporter.ReportProblem(invalidConfigMsg, EventFlowContextIdentifiers.Configuration);
throw new Exception(invalidConfigMsg);
void ensureEventHubName()
{
if (string.IsNullOrWhiteSpace(this.eventHubName))
{
var emptyEventHubNameMsg = $"{nameof(EventHubOutput)}: Event Hub name must not be empty. It can be specified in the '{nameof(EventHubOutputConfiguration.ConnectionString)}' or '{nameof(EventHubOutputConfiguration.EventHubName)}' configuration parameter";
healthReporter.ReportProblem(emptyEventHubNameMsg);
throw new Exception(emptyEventHubNameMsg);
}
}
}
private async Task SendsManyItemsToPartitionedEventHub(int itemCount)
{
var client = new Mock <IEventHubClient>();
var healthReporter = new Mock <IHealthReporter>();
var configuration = new EventHubOutputConfiguration();
configuration.ConnectionString = "Connection string";
configuration.EventHubName = "foo";
configuration.PartitionKeyProperty = "PartitionByProperty";
EventHubOutput eho = new EventHubOutput(configuration, healthReporter.Object, connectionString => client.Object);
//send `itemCount` EventData items with payload of 55Kb+ each and targeting PartitionKey "partition3"
string payload = new string('a', 55000);
await eho.SendEventsAsync(Enumerable.Range(0, itemCount)
.Select(r =>
{
var eventData = new EventData();
eventData.ProviderName = "TestProvider";
eventData.Timestamp = DateTimeOffset.UtcNow;
eventData.Level = LogLevel.Warning;
eventData.Payload.Add("IntProperty", r);
eventData.Payload.Add("StringProperty", payload);
eventData.Payload.Add("PartitionByProperty", "partition3");
return(eventData);
})
.ToList(), 17, CancellationToken.None);
Func <IEnumerable <MessagingEventData>, bool> verifyBatch = batch =>
{
if (batch.Count() != 4)
{
return(false);
}
return(true);
};
//the individual messages are 55Kb in size each, so a maximum of 4 messages total byte size fits within EventHubMessageSizeLimit limits.
//so we expect `itemCount` / 4 batches executed against EventHubClient.SendAsync()
client.Verify(c => c.SendAsync(It.Is <IEnumerable <MessagingEventData> >(b => verifyBatch(b)), "partition3"), Times.Exactly(itemCount / 4));
healthReporter.Verify(hr => hr.ReportWarning(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
healthReporter.Verify(hr => hr.ReportProblem(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
}
public async Task SendsDataToEventHub()
{
var client = new Mock <IEventHubClient>();
var healthReporter = new Mock <IHealthReporter>();
var configuration = new EventHubOutputConfiguration();
configuration.ConnectionString = "Connection string";
configuration.EventHubName = "foo";
EventData e = new EventData();
e.ProviderName = "TestProvider";
e.Timestamp = DateTimeOffset.UtcNow;
e.Level = LogLevel.Warning;
e.Payload.Add("IntProperty", 42);
e.Payload.Add("StringProperty", "perfection");
EventHubOutput eho = new EventHubOutput(configuration, healthReporter.Object, connectionString => client.Object);
await eho.SendEventsAsync(new EventData[] { e, }, 17, CancellationToken.None);
Func <IEnumerable <MessagingEventData>, bool> verifyBatch = batch =>
{
if (batch.Count() != 1)
{
return(false);
}
var data = batch.First();
var dataBytes = data.EventBody.ToArray();
var bodyString = Encoding.UTF8.GetString(dataBytes, (int)data.Offset, dataBytes.Length);
var recordSet = JObject.Parse(bodyString);
var message = recordSet["records"][0];
return((string)message["level"] == "Warning" &&
(string)message["properties"]["ProviderName"] == "TestProvider" &&
(int)message["properties"]["IntProperty"] == 42 &&
(string)message["properties"]["StringProperty"] == "perfection");
};
client.Verify(c => c.SendAsync(It.Is <IEnumerable <MessagingEventData> >(b => verifyBatch(b))), Times.Once);
healthReporter.Verify(hr => hr.ReportWarning(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
healthReporter.Verify(hr => hr.ReportProblem(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
}
public EventHubOutput(IConfiguration configuration, IHealthReporter healthReporter)
{
Requires.NotNull(configuration, nameof(configuration));
Requires.NotNull(healthReporter, nameof(healthReporter));
this.healthReporter = healthReporter;
var eventHubOutputConfiguration = new EventHubOutputConfiguration();
try
{
configuration.Bind(eventHubOutputConfiguration);
}
catch
{
healthReporter.ReportProblem($"Invalid {nameof(EventHubOutput)} configuration encountered: '{configuration.ToString()}'",
EventFlowContextIdentifiers.Configuration);
throw;
}
Initialize(eventHubOutputConfiguration);
}
public async Task MethodInfoIsSerializedAsFullyQualifiedName()
{
var client = new Mock <IEventHubClient>();
var healthReporter = new Mock <IHealthReporter>();
var configuration = new EventHubOutputConfiguration();
configuration.ConnectionString = "Connection string";
configuration.EventHubName = "foo";
EventData e = new EventData();
e.ProviderName = "TestProvider";
e.Timestamp = DateTimeOffset.UtcNow;
e.Level = LogLevel.Warning;
e.Payload.Add("Method", typeof(EventHubOutputTests).GetMethod(nameof(MethodInfoIsSerializedAsFullyQualifiedName)));
EventHubOutput eho = new EventHubOutput(configuration, healthReporter.Object, connectionString => client.Object);
await eho.SendEventsAsync(new EventData[] { e, }, 17, CancellationToken.None);
Func <IEnumerable <MessagingEventData>, bool> verifyBatch = batch =>
{
if (batch.Count() != 1)
{
return(false);
}
var data = batch.First();
var bodyString = Encoding.UTF8.GetString(data.Body.Array, data.Body.Offset, data.Body.Count);
var recordSet = JObject.Parse(bodyString);
var message = recordSet["records"][0];
return((string)message["level"] == "Warning" &&
(string)message["properties"]["ProviderName"] == "TestProvider" &&
(string)message["properties"]["Method"] == "Microsoft.Diagnostics.EventFlow.Outputs.Tests.EventHubOutputTests.MethodInfoIsSerializedAsFullyQualifiedName");
};
client.Verify(c => c.SendAsync(It.Is <IEnumerable <MessagingEventData> >(b => verifyBatch(b))), Times.Once);
healthReporter.Verify(hr => hr.ReportWarning(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
healthReporter.Verify(hr => hr.ReportProblem(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
}
// The Initialize method is not thread-safe. Please only call this on one thread and do so before the pipeline starts sending
// data to this output
private void Initialize(EventHubOutputConfiguration configuration)
{
Debug.Assert(configuration != null);
Debug.Assert(this.healthReporter != null);
if (string.IsNullOrWhiteSpace(configuration.ConnectionString))
{
var errorMessage = $"{nameof(EventHubOutput)}: '{nameof(EventHubOutputConfiguration.ConnectionString)}' configuration parameter must be set to a valid connection string";
healthReporter.ReportProblem(errorMessage, EventFlowContextIdentifiers.Configuration);
throw new Exception(errorMessage);
}
ServiceBusConnectionStringBuilder connStringBuilder = new ServiceBusConnectionStringBuilder(configuration.ConnectionString);
connStringBuilder.TransportType = TransportType.Amqp;
this.eventHubName = connStringBuilder.EntityPath ?? configuration.EventHubName;
if (string.IsNullOrWhiteSpace(this.eventHubName))
{
var errorMessage = $"{nameof(EventHubOutput)}: Event Hub name must not be empty. It can be specified in the '{nameof(EventHubOutputConfiguration.ConnectionString)}' or '{nameof(EventHubOutputConfiguration.EventHubName)}' configuration parameter";
healthReporter.ReportProblem(errorMessage, EventFlowContextIdentifiers.Configuration);
throw new Exception(errorMessage);
}
this.connections = new EventHubConnection[ConcurrentConnections];
// To create a MessageFactory, the connection string can't contain the EntityPath. So we set it to null here.
connStringBuilder.EntityPath = null;
for (uint i = 0; i < this.connections.Length; i++)
{
MessagingFactory factory = MessagingFactory.CreateFromConnectionString(connStringBuilder.ToString());
this.connections[i] = new EventHubConnection();
this.connections[i].MessagingFactory = factory;
this.connections[i].HubClient = factory.CreateEventHubClient(this.eventHubName);
}
}
public async Task SendsDataToPartitionedEventHub()
{
var client = new Mock <IEventHubClient>();
var healthReporter = new Mock <IHealthReporter>();
var configuration = new EventHubOutputConfiguration();
configuration.ConnectionString = "Connection string";
configuration.EventHubName = "foo";
configuration.PartitionKeyProperty = "PartitionByProperty";
EventData e1 = new EventData();
e1.ProviderName = "TestProvider";
e1.Timestamp = DateTimeOffset.UtcNow;
e1.Level = LogLevel.Warning;
e1.Payload.Add("IntProperty", 23);
e1.Payload.Add("StringProperty", "partition-perfection");
e1.Payload.Add("PartitionByProperty", "partition1");
EventData e2 = new EventData();
e2.ProviderName = "TestProvider";
e2.Timestamp = DateTimeOffset.UtcNow;
e2.Level = LogLevel.Warning;
e2.Payload.Add("IntProperty", 23);
e2.Payload.Add("StringProperty", "partition-perfection");
EventData e3 = new EventData();
e3.ProviderName = "TestProvider";
e3.Timestamp = DateTimeOffset.UtcNow;
e3.Level = LogLevel.Warning;
e3.Payload.Add("IntProperty", 23);
e3.Payload.Add("StringProperty", new string('a', 150000));
e3.Payload.Add("PartitionByProperty", "partition2");
EventData e4 = new EventData();
e4.ProviderName = "TestProvider";
e4.Timestamp = DateTimeOffset.UtcNow;
e4.Level = LogLevel.Warning;
e4.Payload.Add("IntProperty", 23);
e4.Payload.Add("StringProperty", new string('b', 150000));
e4.Payload.Add("PartitionByProperty", "partition2");
EventData e5 = new EventData();
e5.ProviderName = "TestProvider";
e5.Timestamp = DateTimeOffset.UtcNow;
e5.Level = LogLevel.Warning;
e5.Payload.Add("IntProperty", 23);
e5.Payload.Add("StringProperty", new string('c', 150000));
e5.Payload.Add("PartitionByProperty", "partition2");
EventHubOutput eho = new EventHubOutput(configuration, healthReporter.Object, connectionString => client.Object);
await eho.SendEventsAsync(new[] { e1, e2, e3, e4, e5 }, 17, CancellationToken.None);
Func <IEnumerable <MessagingEventData>, bool> verifyBatch = batch =>
{
if (batch.Count() != 1)
{
return(false);
}
return(true);
};
client.Verify(c => c.SendAsync(It.Is <IEnumerable <MessagingEventData> >(b => verifyBatch(b))), Times.Once);
client.Verify(c => c.SendAsync(It.Is <IEnumerable <MessagingEventData> >(b => verifyBatch(b)), "partition1"), Times.Once);
client.Verify(c => c.SendAsync(It.Is <IEnumerable <MessagingEventData> >(b => verifyBatch(b)), "partition2"), Times.Exactly(3));
healthReporter.Verify(hr => hr.ReportWarning(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
healthReporter.Verify(hr => hr.ReportProblem(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
}
public async Task UsesCustomJsonSerializerSettings()
{
var client = new Mock <IEventHubClient>();
var healthReporter = new Mock <IHealthReporter>();
var configuration = new EventHubOutputConfiguration();
configuration.ConnectionString = "Connection string";
configuration.EventHubName = "foo";
EventData e = new EventData();
e.ProviderName = "TestProvider";
e.Timestamp = DateTimeOffset.UtcNow;
e.Level = LogLevel.Warning;
e.Payload.Add("InfinityProperty", Double.PositiveInfinity);
EventHubOutput eho = new EventHubOutput(configuration, healthReporter.Object, connectionString => client.Object);
await eho.SendEventsAsync(new EventData[] { e, }, 17, CancellationToken.None);
Func <IEnumerable <MessagingEventData>, bool> verifyBatch = batch =>
{
if (batch.Count() != 1)
{
return(false);
}
var data = batch.First();
var bodyString = Encoding.UTF8.GetString(data.Body.Array, data.Body.Offset, data.Body.Count);
var recordSet = JObject.Parse(bodyString);
var message = recordSet["records"][0];
return((string)message["level"] == "Warning" &&
(string)message["properties"]["ProviderName"] == "TestProvider"
// By default floating-point infinity values are converted to strings
&& (string)message["properties"]["InfinityProperty"] == "Infinity");
};
client.Verify(c => c.SendAsync(It.Is <IEnumerable <MessagingEventData> >(b => verifyBatch(b))), Times.Once);
healthReporter.Verify(hr => hr.ReportWarning(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
healthReporter.Verify(hr => hr.ReportProblem(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
// Now verify changing serializer settings is effective
eho.SerializerSettings.FloatFormatHandling = FloatFormatHandling.DefaultValue;
client.ResetCalls();
verifyBatch = batch =>
{
if (batch.Count() != 1)
{
return(false);
}
var data = batch.First();
var bodyString = Encoding.UTF8.GetString(data.Body.Array, data.Body.Offset, data.Body.Count);
var recordSet = JObject.Parse(bodyString);
var message = recordSet["records"][0];
return((string)message["level"] == "Warning" &&
(string)message["properties"]["ProviderName"] == "TestProvider" &&
(double)message["properties"].OfType <JProperty>().First(p => p.Name == "InfinityProperty").OfType <JValue>().First().Value == 0.0);
};
await eho.SendEventsAsync(new EventData[] { e, }, 18, CancellationToken.None);
client.Verify(c => c.SendAsync(It.Is <IEnumerable <MessagingEventData> >(b => verifyBatch(b))), Times.Once);
healthReporter.Verify(hr => hr.ReportWarning(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
healthReporter.Verify(hr => hr.ReportProblem(It.IsAny <string>(), It.IsAny <string>()), Times.Never);
}