public TableStorageEventListener(IConfigurationProvider configurationProvider, IHealthReporter healthReporter)
: base(configurationProvider, healthReporter)
{
if (this.Disabled)
{
return;
}
Debug.Assert(configurationProvider != null);
this.CreateTableClient(configurationProvider);
Random randomNumberGenerator = new Random();
this.instanceId = randomNumberGenerator.Next(100000000).ToString("D8");
this.nextEntityId = 0;
this.identityIdResetLock = new object();
this.Sender = new ConcurrentEventSender<EventData>(
eventBufferSize: 1000,
maxConcurrency: MaxConcurrentPartitions,
batchSize: 50,
noEventsDelay: TimeSpan.FromMilliseconds(1000),
transmitterProc: this.SendEventsAsync,
healthReporter: healthReporter);
}
public BufferingEventListener(IConfigurationProvider configurationProvider, IHealthReporter healthReporter)
{
if (configurationProvider == null)
{
throw new ArgumentNullException("configurationProvider");
}
if (healthReporter == null)
{
throw new ArgumentNullException("healthReporter");
}
this.healthReporter = healthReporter;
this.errorReportingThrottle = new TimeSpanThrottle(TimeSpan.FromSeconds(1));
this.Disabled = !configurationProvider.HasConfiguration;
}
public EventHubListener(IConfigurationProvider configurationProvider, IHealthReporter healthReporter) : base(configurationProvider, healthReporter)
{
if (this.Disabled)
{
return;
}
Debug.Assert(configurationProvider != null);
this.CreateConnectionData(configurationProvider);
this.Sender = new ConcurrentEventSender<EventData>(
eventBufferSize: 1000,
maxConcurrency: ConcurrentConnections,
batchSize: 50,
noEventsDelay: TimeSpan.FromMilliseconds(1000),
transmitterProc: this.SendEventsAsync,
healthReporter: healthReporter);
}
public ApplicationInsightsEventListener(IConfigurationProvider configurationProvider, IHealthReporter healthReporter) : base(configurationProvider, healthReporter)
{
if (this.Disabled)
{
return;
}
Debug.Assert(configurationProvider != null);
telemetry = new TelemetryClient();
telemetry.Context.InstrumentationKey = configurationProvider.GetValue(AppInsightsKeyName);
this.Sender = new ConcurrentEventSender<EventData>(
eventBufferSize: 1000,
maxConcurrency: 2,
batchSize: 100,
noEventsDelay: TimeSpan.FromMilliseconds(1000),
transmitterProc: this.SendEventsAsync,
healthReporter: healthReporter);
}
public BufferingEventListener(IConfigurationProvider configurationProvider, IHealthReporter healthReporter)
{
if (configurationProvider == null)
{
throw new ArgumentNullException("configurationProvider");
}
if (healthReporter == null)
{
throw new ArgumentNullException("healthReporter");
}
this.healthReporter = healthReporter;
this.errorReportingThrottle = new TimeSpanThrottle(TimeSpan.FromSeconds(1));
lock (this) // See OnEventSourceCreated() for explanation why we are locking on 'this' here.
{
this.Disabled = !configurationProvider.HasConfiguration;
EnableInitialSources();
this.constructed = true;
}
}
public StdOutput CreateItem(IConfiguration configuration, IHealthReporter healthReporter)
{
Requires.NotNull(healthReporter, nameof(healthReporter));
return(new StdOutput(healthReporter));
}
public static EventData ToEventData(this EventWrittenEventArgs eventSourceEvent, IHealthReporter healthReporter, string context)
{
Debug.Assert(healthReporter != null);
// High-precision event timestamping is availabe on .NET 4.6+ and .NET Core 2.0+
// For the latter the implementation of DateTime.UtcNow has changed and we do not need to do anything.
// .NET Core 1.1 will use imprecise timestamp--there is no easy fix for this target.
#if NETSTANDARD1_6
DateTime now = DateTime.UtcNow;
#else
DateTime now;
if (hasPreciseTime)
{
GetSystemTimePreciseAsFileTime(out long filetime);
now = DateTime.FromFileTimeUtc(filetime);
}
else
{
now = DateTime.UtcNow;
}
#endif
EventData eventData = new EventData
{
ProviderName = eventSourceEvent.EventSource.Name,
Timestamp = now,
Level = (LogLevel)(int)eventSourceEvent.Level,
Keywords = (long)eventSourceEvent.Keywords
};
IDictionary <string, object> payloadData = eventData.Payload;
payloadData.Add(nameof(eventSourceEvent.EventId), eventSourceEvent.EventId);
payloadData.Add(nameof(eventSourceEvent.EventName), eventSourceEvent.EventName);
if (eventSourceEvent.ActivityId != default(Guid))
{
payloadData.Add(nameof(EventWrittenEventArgs.ActivityId), ActivityPathDecoder.GetActivityPathString(eventSourceEvent.ActivityId));
}
if (eventSourceEvent.RelatedActivityId != default(Guid))
{
payloadData.Add(nameof(EventWrittenEventArgs.RelatedActivityId), ActivityPathDecoder.GetActivityPathString(eventSourceEvent.RelatedActivityId));
}
try
{
if (eventSourceEvent.Message != null)
{
// If the event has a badly formatted manifest, the FormattedMessage property getter might throw
payloadData.Add(nameof(eventSourceEvent.Message), string.Format(CultureInfo.InvariantCulture, eventSourceEvent.Message, eventSourceEvent.Payload.ToArray()));
}
}
catch { }
eventSourceEvent.ExtractPayloadData(eventData, healthReporter, context);
return(eventData);
}
public DiagnosticPipeline(
IHealthReporter healthReporter,
IReadOnlyCollection <IObservable <EventData> > inputs,
IReadOnlyCollection <IFilter> globalFilters,
IReadOnlyCollection <EventSink> sinks,
DiagnosticPipelineConfiguration pipelineConfiguration = null,
bool disposeDependencies = false,
TaskScheduler taskScheduler = null)
{
Requires.NotNull(healthReporter, nameof(healthReporter));
Requires.NotNull(inputs, nameof(inputs));
Requires.Argument(inputs.Count > 0, nameof(inputs), "There must be at least one input");
Requires.NotNull(sinks, nameof(sinks));
Requires.Argument(sinks.Count > 0, nameof(sinks), "There must be at least one sink");
this.pipelineConfiguration = pipelineConfiguration ?? new DiagnosticPipelineConfiguration();
taskScheduler = taskScheduler ?? TaskScheduler.Current;
// An estimatie how many batches of events to allow inside the pipeline.
// We want to be able to process full buffer of events, but also have enough batches in play in case of high concurrency.
int MaxNumberOfBatchesInProgress = Math.Max(
5 * this.pipelineConfiguration.MaxConcurrency,
this.pipelineConfiguration.PipelineBufferSize / this.pipelineConfiguration.MaxEventBatchSize);
this.Inputs = inputs;
this.Sinks = sinks;
// Just play nice and make sure there is always something to enumerate on
this.GlobalFilters = globalFilters ?? new IFilter[0];
this.HealthReporter = healthReporter;
this.cancellationTokenSource = new CancellationTokenSource();
var propagateCompletion = new DataflowLinkOptions()
{
PropagateCompletion = true
};
// One disposable for each input subscription.
this.inputSubscriptions = new List <IDisposable>(inputs.Count);
var inputBuffer = new BufferBlock <EventData>(
new DataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.PipelineBufferSize,
CancellationToken = this.cancellationTokenSource.Token,
TaskScheduler = taskScheduler
});
this.pipelineHead = inputBuffer;
var batcher = new BatchBlock <EventData>(
this.pipelineConfiguration.MaxEventBatchSize,
new GroupingDataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.PipelineBufferSize,
CancellationToken = this.cancellationTokenSource.Token,
TaskScheduler = taskScheduler
}
);
inputBuffer.LinkTo(batcher, propagateCompletion);
this.batcherTimer = new Timer(
(unused) => { try { batcher.TriggerBatch(); } catch {} },
state: null,
dueTime: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec),
period: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec));
ISourceBlock <EventData[]> sinkSource;
FilterAction filterTransform;
if (this.GlobalFilters.Count > 0)
{
filterTransform = new FilterAction(
this.GlobalFilters,
this.cancellationTokenSource.Token,
MaxNumberOfBatchesInProgress,
this.pipelineConfiguration.MaxConcurrency,
healthReporter,
taskScheduler);
var globalFiltersBlock = filterTransform.GetFilterBlock();
batcher.LinkTo(globalFiltersBlock, propagateCompletion);
sinkSource = globalFiltersBlock;
}
else
{
sinkSource = batcher;
}
bool usingBroadcastBlock = sinks.Count > 1;
if (usingBroadcastBlock)
{
var broadcaster = new BroadcastBlock <EventData[]>(
(events) => events?.Select((e) => e.DeepClone()).ToArray(),
new DataflowBlockOptions()
{
BoundedCapacity = MaxNumberOfBatchesInProgress,
CancellationToken = this.cancellationTokenSource.Token,
TaskScheduler = taskScheduler
});
sinkSource.LinkTo(broadcaster, propagateCompletion);
sinkSource = broadcaster;
}
this.outputCompletionTasks = new List <Task>(sinks.Count);
foreach (var sink in sinks)
{
ISourceBlock <EventData[]> outputSource = sinkSource;
if (sink.Filters != null && sink.Filters.Count > 0)
{
filterTransform = new FilterAction(
sink.Filters,
this.cancellationTokenSource.Token,
MaxNumberOfBatchesInProgress,
this.pipelineConfiguration.MaxConcurrency,
healthReporter,
taskScheduler);
var filterBlock = filterTransform.GetFilterBlock();
if (usingBroadcastBlock)
{
var lossReportingPropagator = new LossReportingPropagatorBlock <EventData[]>(this.HealthReporter);
sinkSource.LinkTo(lossReportingPropagator, propagateCompletion);
lossReportingPropagator.LinkTo(filterBlock, propagateCompletion);
}
else
{
sinkSource.LinkTo(filterBlock, propagateCompletion);
}
outputSource = filterBlock;
}
else if (usingBroadcastBlock)
{
var lossReportingPropagator = new LossReportingPropagatorBlock <EventData[]>(this.HealthReporter);
sinkSource.LinkTo(lossReportingPropagator, propagateCompletion);
outputSource = lossReportingPropagator;
}
OutputAction outputAction = new OutputAction(
sink.Output,
this.cancellationTokenSource.Token,
MaxNumberOfBatchesInProgress,
this.pipelineConfiguration.MaxConcurrency,
healthReporter,
taskScheduler);
var outputBlock = outputAction.GetOutputBlock();
outputSource.LinkTo(outputBlock, propagateCompletion);
this.outputCompletionTasks.Add(outputBlock.Completion);
}
IObserver <EventData> inputBufferObserver = new TargetBlockObserver <EventData>(inputBuffer, this.HealthReporter);
foreach (var input in inputs)
{
this.inputSubscriptions.Add(input.Subscribe(inputBufferObserver));
}
this.disposed = false;
this.disposeDependencies = disposeDependencies;
}
private DiagnosticPipeline createPipeline(LoggerInput input, IHealthReporter reporter)
{
return(new DiagnosticPipeline(reporter, new[] { input }, null, new[] { new EventSink(new FakeOutput(), null) }));
}
private static void CreateItemFactories(
IConfiguration configuration,
IHealthReporter healthReporter,
out IDictionary <string, string> inputFactories,
out IDictionary <string, string> outputFactories,
out IDictionary <string, string> filterFactories)
{
Debug.Assert(configuration != null);
Debug.Assert(healthReporter != null);
inputFactories = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
inputFactories["EventSource"] = "Microsoft.Diagnostics.EventFlow.Inputs.EventSourceInputFactory, Microsoft.Diagnostics.EventFlow.Inputs.EventSource, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
inputFactories["PerformanceCounter"] = "Microsoft.Diagnostics.EventFlow.Inputs.PerformanceCounterInputFactory, Microsoft.Diagnostics.EventFlow.Inputs.PerformanceCounter, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
inputFactories["Trace"] = "Microsoft.Diagnostics.EventFlow.Inputs.TraceInputFactory, Microsoft.Diagnostics.EventFlow.Inputs.Trace, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
inputFactories["Serilog"] = "Microsoft.Diagnostics.EventFlow.Inputs.SerilogInputFactory, Microsoft.Diagnostics.EventFlow.Inputs.Serilog, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
outputFactories = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
outputFactories["ApplicationInsights"] = "Microsoft.Diagnostics.EventFlow.Outputs.ApplicationInsightsOutputFactory, Microsoft.Diagnostics.EventFlow.Outputs.ApplicationInsights, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
outputFactories["StdOutput"] = "Microsoft.Diagnostics.EventFlow.Outputs.StdOutputFactory, Microsoft.Diagnostics.EventFlow.Outputs.StdOutput, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
outputFactories["EventHub"] = "Microsoft.Diagnostics.EventFlow.Outputs.EventHubOutputFactory, Microsoft.Diagnostics.EventFlow.Outputs.EventHub, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
outputFactories["ElasticSearch"] = "Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearchOutputFactory, Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
outputFactories["OmsOutput"] = "Microsoft.Diagnostics.EventFlow.Outputs.OmsOutputFactory, Microsoft.Diagnostics.EventFlow.Outputs.Oms, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
filterFactories = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
filterFactories["metadata"] = "Microsoft.Diagnostics.EventFlow.Filters.EventMetadataFilterFactory, Microsoft.Diagnostics.EventFlow.Core, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
filterFactories["drop"] = "Microsoft.Diagnostics.EventFlow.Filters.DropFilterFactory, Microsoft.Diagnostics.EventFlow.Core, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
// read 3rd party plugins
IConfiguration extensionsConfiguration = configuration.GetSection("extensions");
foreach (var extension in extensionsConfiguration.GetChildren())
{
var extConfig = new ExtensionsConfiguration();
extension.Bind(extConfig);
IDictionary <string, string> targetFactories = null;
if (string.Equals(extConfig.Category, ExtensionCategories.InputFactory, StringComparison.OrdinalIgnoreCase))
{
targetFactories = inputFactories;
}
else if (string.Equals(extConfig.Category, ExtensionCategories.OutputFactory, StringComparison.OrdinalIgnoreCase))
{
targetFactories = outputFactories;
}
else if (string.Equals(extConfig.Category, ExtensionCategories.FilterFactory, StringComparison.OrdinalIgnoreCase))
{
targetFactories = filterFactories;
}
else if (string.Equals(extConfig.Category, ExtensionCategories.HealthReporter, StringComparison.OrdinalIgnoreCase))
{
// Health reporter should have been created earlier, so skip
continue;
}
else
{
healthReporter.ReportWarning($"Unrecognized extension category: {extConfig.Category}");
continue;
}
if (!string.IsNullOrEmpty(extConfig.Type) && !string.IsNullOrEmpty(extConfig.QualifiedTypeName))
{
targetFactories[extConfig.Type] = extConfig.QualifiedTypeName;
}
else
{
healthReporter.ReportWarning($"Invalid extension configuration is skipped. Category: {extConfig.Category}, Type: {extConfig.Type}, QualifiedTypeName: {extConfig.QualifiedTypeName}");
}
}
}
private static List <ItemWithChildren <PipelineItemType, PipelineItemChildType> > ProcessSection <PipelineItemType, PipelineItemChildType>(
IConfigurationSection configurationSection,
IHealthReporter healthReporter,
IDictionary <string, string> itemFactories,
IDictionary <string, string> childFactories,
string childSectionName)
{
Debug.Assert(!string.IsNullOrWhiteSpace(configurationSection.Key));
Debug.Assert(healthReporter != null);
Debug.Assert(itemFactories != null);
Debug.Assert((string.IsNullOrEmpty(childSectionName) && childFactories == null) || (!string.IsNullOrEmpty(childSectionName) && childFactories != null));
List <ItemWithChildren <PipelineItemType, PipelineItemChildType> > createdItems = new List <ItemWithChildren <PipelineItemType, PipelineItemChildType> >();
if (configurationSection == null)
{
return(createdItems);
}
List <IConfigurationSection> itemConfigurationFragments = configurationSection.GetChildren().ToList();
foreach (var itemFragment in itemConfigurationFragments)
{
ItemConfiguration itemConfiguration = new ItemConfiguration();
try
{
itemFragment.Bind(itemConfiguration);
}
catch
{
ReportInvalidConfigurationFragmentAndThrow(healthReporter, itemFragment);
}
string itemFactoryTypeName;
if (!itemFactories.TryGetValue(itemConfiguration.Type, out itemFactoryTypeName))
{
ReportUnknownItemTypeAndThrow(healthReporter, configurationSection, itemConfiguration);
}
IPipelineItemFactory <PipelineItemType> factory;
PipelineItemType item = default(PipelineItemType);
try
{
var itemFactoryType = Type.GetType(itemFactoryTypeName, throwOnError: true);
factory = Activator.CreateInstance(itemFactoryType) as IPipelineItemFactory <PipelineItemType>;
item = factory.CreateItem(itemFragment, healthReporter);
}
catch (Exception e)
{
ReportItemCreationFailedAndThrow(healthReporter, itemConfiguration.Type, e);
}
// The factory will do its own error reporting, so if it returns null, no further error reporting is necessary.
if (item == null)
{
continue;
}
List <ItemWithChildren <PipelineItemChildType, object> > children = null;
if (!string.IsNullOrEmpty(childSectionName))
{
IConfigurationSection childrenSection = itemFragment.GetSection(childSectionName);
children = ProcessSection <PipelineItemChildType, object>(
childrenSection,
healthReporter,
childFactories,
childFactories: null, // Only one level of nexting is supported
childSectionName: null);
createdItems.Add(new ItemWithChildren <PipelineItemType, PipelineItemChildType>(item, children.Select(c => c.Item).ToList()));
}
else
{
createdItems.Add(new ItemWithChildren <PipelineItemType, PipelineItemChildType>(item, null));
}
}
return(createdItems);
}
public HealthController(IHealthReporter healthReporter)
{
this.healthReporter = healthReporter;
}
private void Initialize(IReadOnlyCollection <EventSourceConfiguration> eventSources, IHealthReporter healthReporter)
{
this.healthReporter = healthReporter;
this.subject = new EventFlowSubject <EventData>();
this.EventSources = eventSources;
if (this.EventSources.Count() == 0)
{
healthReporter.ReportWarning($"{nameof(EventSourceInput)}: no event sources configured", nameof(EventSourceInput));
return;
}
lock (this) // See OnEventSourceCreated() for explanation why we are locking on 'this' here.
{
EnableInitialSources();
this.constructed = true;
}
}
public LoggerInput CreateItem(IConfiguration configuration, IHealthReporter healthReporter)
{
Validation.Requires.NotNull(healthReporter, nameof(healthReporter));
return(new LoggerInput(healthReporter));
}
public UpdateAppDomainName CreateItem(IConfiguration configuration, IHealthReporter healthReporter)
{
return(new UpdateAppDomainName());
}
public ApplicationInsightsOutput(ApplicationInsightsOutputConfiguration applicationInsightsOutputConfiguration, IHealthReporter healthReporter)
{
Requires.NotNull(applicationInsightsOutputConfiguration, nameof(applicationInsightsOutputConfiguration));
Requires.NotNull(healthReporter, nameof(healthReporter));
this.healthReporter = healthReporter;
Initialize(applicationInsightsOutputConfiguration);
}
public static void AddPayloadProperty(IDictionary <string, object> payload, string key, object value, IHealthReporter healthReporter, string context)
{
Requires.NotNull(payload, nameof(payload));
Requires.NotNull(key, nameof(key));
Requires.NotNull(healthReporter, nameof(healthReporter));
if (!payload.ContainsKey(key))
{
payload.Add(key, value);
return;
}
string newKey;
int i = 1;
//update property key till there is no such key in dict
do
{
newKey = key + "_" + i.ToString("d");
i++;
}while (payload.ContainsKey(newKey));
payload.Add(newKey, value);
healthReporter.ReportWarning($"The property with the key '{key}' already exist in the event payload. Value was added under key '{newKey}'", context);
}
public EtwInputConfigurationPatch(IConfiguration configuration, IHealthReporter healthReporter)
: base(configuration, healthReporter)
{
}
public EventSourceInput(IReadOnlyCollection <EventSourceConfiguration> eventSources, IHealthReporter healthReporter)
{
Requires.NotNull(eventSources, nameof(eventSources));
Requires.NotNull(healthReporter, nameof(healthReporter));
Initialize(eventSources, healthReporter);
}
internal static IConfigurationRoot ApplyFabricConfigurationOverrides(
this IConfigurationRoot configurationRoot,
string configPackagePath,
IHealthReporter healthReporter)
{
Debug.Assert(configurationRoot != null);
Debug.Assert(!string.IsNullOrWhiteSpace(configPackagePath));
Debug.Assert(healthReporter != null);
Regex fabricValueReferenceRegex = new Regex(FabricConfigurationValueReference, RegexOptions.CultureInvariant, TimeSpan.FromMilliseconds(100));
Regex fabricFileReferenceRegex = new Regex(FabricConfigurationFileReference, RegexOptions.CultureInvariant, TimeSpan.FromMilliseconds(100));
// Use ToList() to ensure that configuration is fully enumerated before starting to modify it.
foreach (var kvp in configurationRoot.AsEnumerable().ToList())
{
if (kvp.Value == null)
{
continue;
}
try
{
Match valueReferenceMatch = fabricValueReferenceRegex.Match(kvp.Value);
if (valueReferenceMatch.Success)
{
string valueReferencePath = ConfigurationPath.Combine(valueReferenceMatch.Groups["section"].Value, valueReferenceMatch.Groups["name"].Value);
string newValue = configurationRoot[valueReferencePath];
if (string.IsNullOrEmpty(newValue))
{
healthReporter.ReportWarning(
$"Configuration value reference '{kvp.Value}' was encountered but no corresponding configuration value was found using path '{valueReferencePath}'",
EventFlowContextIdentifiers.Configuration);
}
else
{
configurationRoot[kvp.Key] = newValue;
}
}
Match fileReferenceMatch = fabricFileReferenceRegex.Match(kvp.Value);
if (fileReferenceMatch.Success)
{
string configFileName = fileReferenceMatch.Groups["filename"].Value;
if (string.IsNullOrWhiteSpace(configFileName))
{
healthReporter.ReportWarning(
$"Configuration file reference '{kvp.Value}' was encountered but the file name part is missing",
EventFlowContextIdentifiers.Configuration);
}
else
{
string configFilePath = Path.Combine(configPackagePath, configFileName);
configurationRoot[kvp.Key] = configFilePath;
}
}
}
catch (RegexMatchTimeoutException)
{
healthReporter.ReportWarning(
$"Configuration entry with key '{kvp.Key}' and value '{kvp.Value}' could not be checked if it represents a configuration value reference--a timeout occurred when the value was being parsed.",
EventFlowContextIdentifiers.Configuration);
continue;
}
}
return(configurationRoot);
}
public static EventData ToEventData(this EventWrittenEventArgs eventSourceEvent, IHealthReporter healthReporter, string context)
{
Debug.Assert(healthReporter != null);
EventData eventData = new EventData
{
ProviderName = eventSourceEvent.EventSource.Name,
Timestamp = DateTimePrecise.UtcNow,
Level = (LogLevel)(int)eventSourceEvent.Level,
Keywords = (long)eventSourceEvent.Keywords
};
IDictionary <string, object> payloadData = eventData.Payload;
payloadData.Add(nameof(eventSourceEvent.EventId), eventSourceEvent.EventId);
payloadData.Add(nameof(eventSourceEvent.EventName), eventSourceEvent.EventName);
if (eventSourceEvent.ActivityId != default(Guid))
{
payloadData.Add(nameof(EventWrittenEventArgs.ActivityId), ActivityPathDecoder.GetActivityPathString(eventSourceEvent.ActivityId));
}
if (eventSourceEvent.RelatedActivityId != default(Guid))
{
payloadData.Add(nameof(EventWrittenEventArgs.RelatedActivityId), ActivityPathDecoder.GetActivityPathString(eventSourceEvent.RelatedActivityId));
}
try
{
if (eventSourceEvent.Message != null)
{
// If the event has a badly formatted manifest, the FormattedMessage property getter might throw
payloadData.Add(nameof(eventSourceEvent.Message), string.Format(CultureInfo.InvariantCulture, eventSourceEvent.Message, eventSourceEvent.Payload.ToArray()));
}
}
catch { }
eventSourceEvent.ExtractPayloadData(eventData, healthReporter, context);
return(eventData);
}
public DiagnosticPipeline(
IHealthReporter healthReporter,
IReadOnlyCollection <IObservable <EventData> > inputs,
IReadOnlyCollection <IFilter> globalFilters,
IReadOnlyCollection <EventSink> sinks,
DiagnosticPipelineConfiguration pipelineConfiguration = null,
bool disposeDependencies = false)
{
Requires.NotNull(healthReporter, nameof(healthReporter));
Requires.NotNull(inputs, nameof(inputs));
Requires.Argument(inputs.Count > 0, nameof(inputs), "There must be at least one input");
Requires.NotNull(sinks, nameof(sinks));
Requires.Argument(sinks.Count > 0, nameof(sinks), "There must be at least one sink");
this.eventsInProgress = 0;
this.pipelineConfiguration = pipelineConfiguration ?? new DiagnosticPipelineConfiguration();
// An estimatie how many batches of events to allow inside the pipeline.
// We want to be able to process full buffer of events, but also have enough batches in play in case of high concurrency.
int MaxNumberOfBatchesInProgress = Math.Max(
5 * this.pipelineConfiguration.MaxConcurrency,
this.pipelineConfiguration.PipelineBufferSize / this.pipelineConfiguration.MaxEventBatchSize);
this.Inputs = inputs;
this.Sinks = sinks;
// Just play nice and make sure there is always something to enumerate on
this.GlobalFilters = globalFilters ?? new IFilter[0];
this.HealthReporter = healthReporter;
this.cancellationTokenSource = new CancellationTokenSource();
var propagateCompletion = new DataflowLinkOptions()
{
PropagateCompletion = true
};
this.pipelineLinkDisposables = new List <IDisposable>();
this.pipelineCompletionTasks = new List <Task>();
var inputBuffer = new BufferBlock <EventData>(
new DataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.PipelineBufferSize,
CancellationToken = this.cancellationTokenSource.Token
});
this.pipelineHead = inputBuffer;
this.pipelineCompletionTasks.Add(inputBuffer.Completion);
var batcher = new BatchBlock <EventData>(
this.pipelineConfiguration.MaxEventBatchSize,
new GroupingDataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.MaxEventBatchSize,
CancellationToken = this.cancellationTokenSource.Token
}
);
this.pipelineLinkDisposables.Add(inputBuffer.LinkTo(batcher, propagateCompletion));
this.pipelineCompletionTasks.Add(batcher.Completion);
this.pipelineLinkDisposables.Add(new Timer(
(unused) => batcher.TriggerBatch(),
state: null,
dueTime: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec),
period: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec)));
ISourceBlock <EventData[]> sinkSource;
FilterAction filterTransform;
if (this.GlobalFilters.Count > 0)
{
filterTransform = new FilterAction(
this.GlobalFilters,
this.cancellationTokenSource.Token,
this.pipelineConfiguration.MaxEventBatchSize,
this.pipelineConfiguration.MaxConcurrency,
healthReporter,
this.OnEventsFilteredOut);
var globalFiltersBlock = filterTransform.GetFilterBlock();
this.pipelineLinkDisposables.Add(batcher.LinkTo(globalFiltersBlock, propagateCompletion));
this.pipelineCompletionTasks.Add(globalFiltersBlock.Completion);
sinkSource = globalFiltersBlock;
}
else
{
sinkSource = batcher;
}
if (sinks.Count > 1)
{
// After broadcasting we will effectively have (sinks.Count - 1) * batch.Length more events in the pipeline,
// because the broadcaster is cloning the events for the sake of each sink (filters-output combination).
var eventCounter = new TransformBlock <EventData[], EventData[]>(
(batch) => { Interlocked.Add(ref this.eventsInProgress, (sinks.Count - 1) * batch.Length); return(batch); },
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = MaxNumberOfBatchesInProgress,
CancellationToken = this.cancellationTokenSource.Token
});
this.pipelineLinkDisposables.Add(sinkSource.LinkTo(eventCounter, propagateCompletion));
this.pipelineCompletionTasks.Add(eventCounter.Completion);
var broadcaster = new BroadcastBlock <EventData[]>(
(events) => events?.Select((e) => e.DeepClone()).ToArray(),
new DataflowBlockOptions()
{
BoundedCapacity = MaxNumberOfBatchesInProgress,
CancellationToken = this.cancellationTokenSource.Token
});
this.pipelineLinkDisposables.Add(eventCounter.LinkTo(broadcaster, propagateCompletion));
this.pipelineCompletionTasks.Add(broadcaster.Completion);
sinkSource = broadcaster;
}
foreach (var sink in sinks)
{
ISourceBlock <EventData[]> outputSource = sinkSource;
if (sink.Filters != null && sink.Filters.Count > 0)
{
filterTransform = new FilterAction(
sink.Filters,
this.cancellationTokenSource.Token,
MaxNumberOfBatchesInProgress,
this.pipelineConfiguration.MaxConcurrency,
healthReporter,
this.OnEventsFilteredOut);
var filterBlock = filterTransform.GetFilterBlock();
this.pipelineLinkDisposables.Add(sinkSource.LinkTo(filterBlock, propagateCompletion));
this.pipelineCompletionTasks.Add(filterBlock.Completion);
outputSource = filterBlock;
}
OutputAction outputAction = new OutputAction(
sink.Output,
this.cancellationTokenSource.Token,
MaxNumberOfBatchesInProgress,
this.pipelineConfiguration.MaxConcurrency,
healthReporter,
(eventsSentCount) => Interlocked.Add(ref this.eventsInProgress, -eventsSentCount));
var outputBlock = outputAction.GetOutputBlock();
this.pipelineLinkDisposables.Add(outputSource.LinkTo(outputBlock, propagateCompletion));
this.pipelineCompletionTasks.Add(outputBlock.Completion);
}
IObserver <EventData> inputBufferObserver = new TargetBlockObserver <EventData>(
inputBuffer,
this.HealthReporter,
() => Interlocked.Increment(ref this.eventsInProgress));
this.inputSubscriptions = new List <IDisposable>(inputs.Count);
foreach (var input in inputs)
{
this.inputSubscriptions.Add(input.Subscribe(inputBufferObserver));
}
this.disposed = false;
this.disposeDependencies = disposeDependencies;
}
private static void ExtractPayloadData(this EventWrittenEventArgs eventSourceEvent, EventData eventData, IHealthReporter healthReporter, string context)
{
if (eventSourceEvent.Payload == null || eventSourceEvent.PayloadNames == null)
{
return;
}
bool isEventCountersEvent = eventSourceEvent.EventName == "EventCounters" && eventSourceEvent.Payload.Count == 1 && eventSourceEvent.PayloadNames[0] == "Payload";
if (isEventCountersEvent)
{
ExtractEventCounterPayloadData(eventSourceEvent, eventData, healthReporter, context);
}
else
{
ExtractEventPayloadData(eventSourceEvent, eventData, healthReporter, context);
}
}
private static void ReportInvalidPipelineConfiguration(IHealthReporter healthReporter)
{
var errMsg = $"{nameof(DiagnosticPipelineFactory)}: pipeline settings configuration section is invalid--will use default settings for the diagnostic pipeline";
healthReporter.ReportWarning(errMsg);
}
private static void ExtractEventPayloadData(this EventWrittenEventArgs eventSourceEvent, EventData eventData, IHealthReporter healthReporter, string context)
{
IEnumerator <object> payloadEnumerator = eventSourceEvent.Payload.GetEnumerator();
IEnumerator <string> payloadNamesEnunmerator = eventSourceEvent.PayloadNames.GetEnumerator();
while (payloadEnumerator.MoveNext())
{
payloadNamesEnunmerator.MoveNext();
eventData.AddPayloadProperty(payloadNamesEnunmerator.Current, payloadEnumerator.Current, healthReporter, context);
}
}
public static DiagnosticPipeline CreatePipeline(IConfiguration configuration, IHealthReporter healthReporter = null)
{
Requires.NotNull(configuration, nameof(configuration));
if (healthReporter == null)
{
healthReporter = CreateHealthReporter(configuration);
}
Verify.Operation(healthReporter != null, $"An instance of a health reporter could not be created and none was provider as a parameter to {nameof(CreatePipeline)} method");
(healthReporter as IRequireActivation)?.Activate();
IDictionary <string, string> inputFactories;
IDictionary <string, string> outputFactories;
IDictionary <string, string> filterFactories;
CreateItemFactories(configuration, healthReporter, out inputFactories, out outputFactories, out filterFactories);
// Step 1: instantiate inputs
IConfigurationSection inputConfigurationSection = configuration.GetSection("inputs");
if (inputConfigurationSection.GetChildren().Count() == 0)
{
ReportSectionEmptyAndThrow(healthReporter, inputConfigurationSection);
}
List <ItemWithChildren <IObservable <EventData>, object> > inputCreationResult;
inputCreationResult = ProcessSection <IObservable <EventData>, object>(
inputConfigurationSection,
healthReporter,
inputFactories,
childFactories: null,
childSectionName: null);
List <IObservable <EventData> > inputs = inputCreationResult.Select(item => item.Item).ToList();
if (inputs.Count == 0)
{
ReportNoItemsCreatedAndThrow(healthReporter, inputConfigurationSection);
}
try
{
// Step 2: instantiate global filters (if any)
IConfigurationSection globalFilterConfigurationSection = configuration.GetSection("filters");
List <ItemWithChildren <IFilter, object> > globalFilterCreationResult;
// It completely fine to have a pipeline with no globals filters section, or an empty one
globalFilterCreationResult = ProcessSection <IFilter, object>(
globalFilterConfigurationSection,
healthReporter,
filterFactories,
childFactories: null,
childSectionName: null);
List <IFilter> globalFilters = globalFilterCreationResult.Select(item => item.Item).ToList();
// Step 3: instantiate outputs
IConfigurationSection outputConfigurationSection = configuration.GetSection("outputs");
if (outputConfigurationSection.GetChildren().Count() == 0)
{
ReportSectionEmptyAndThrow(healthReporter, outputConfigurationSection);
}
List <ItemWithChildren <IOutput, IFilter> > outputCreationResult;
outputCreationResult = ProcessSection <IOutput, IFilter>(
outputConfigurationSection,
healthReporter,
outputFactories,
filterFactories,
childSectionName: "filters");
List <IOutput> outputs = outputCreationResult.Select(item => item.Item).ToList();
if (outputs.Count == 0)
{
ReportNoItemsCreatedAndThrow(healthReporter, outputConfigurationSection);
}
// Step 4: assemble and return the pipeline
IReadOnlyCollection <EventSink> sinks = outputCreationResult.Select(outputResult =>
new EventSink(outputResult.Item, outputResult.Children)
).ToList();
var pipelineSettings = new DiagnosticPipelineConfiguration();
IConfigurationSection settingsConfigurationSection = configuration.GetSection("settings");
try
{
if (settingsConfigurationSection.GetChildren().Count() != 0)
{
settingsConfigurationSection.Bind(pipelineSettings);
}
}
catch
{
ReportInvalidPipelineConfiguration(healthReporter);
}
DiagnosticPipeline pipeline = new DiagnosticPipeline(healthReporter, inputs, globalFilters, sinks, pipelineSettings, disposeDependencies: true);
// Now the pipeline has assumed ownership of the inputs, setting inputs variable back to null so we won't
// incorrectly dispose it in the finally block
inputs = null;
return(pipeline);
}
finally
{
DisposeOf(inputs);
}
}
public DiagnosticPipeline(
IHealthReporter healthReporter,
IReadOnlyCollection <IObservable <EventData> > inputs,
IReadOnlyCollection <IFilter> globalFilters,
IReadOnlyCollection <EventSink> sinks,
DiagnosticPipelineConfiguration pipelineConfiguration = null,
bool disposeDependencies = false)
{
Requires.NotNull(healthReporter, nameof(healthReporter));
Requires.NotNull(inputs, nameof(inputs));
Requires.Argument(inputs.Count > 0, nameof(inputs), "There must be at least one input");
Requires.NotNull(sinks, nameof(sinks));
Requires.Argument(sinks.Count > 0, nameof(sinks), "There must be at least one sink");
this.pipelineConfiguration = pipelineConfiguration ?? new DiagnosticPipelineConfiguration();
this.Inputs = inputs;
this.Sinks = sinks;
// Just play nice and make sure there is always something to enumerate on
this.GlobalFilters = globalFilters ?? new IFilter[0];
this.HealthReporter = healthReporter;
this.cancellationTokenSource = new CancellationTokenSource();
var propagateCompletion = new DataflowLinkOptions()
{
PropagateCompletion = true
};
this.pipelineDisposables = new List <IDisposable>();
var inputBuffer = new BufferBlock <EventData>(
new DataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.PipelineBufferSize,
CancellationToken = this.cancellationTokenSource.Token
});
this.pipelineHead = inputBuffer;
var batcher = new BatchBlock <EventData>(
this.pipelineConfiguration.MaxEventBatchSize,
new GroupingDataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.MaxEventBatchSize,
CancellationToken = this.cancellationTokenSource.Token
}
);
this.pipelineDisposables.Add(inputBuffer.LinkTo(batcher, propagateCompletion));
this.pipelineDisposables.Add(new Timer(
(unused) => batcher.TriggerBatch(),
state: null,
dueTime: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec),
period: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec)));
ISourceBlock <EventData[]> sinkSource;
FilterAction filterTransform;
if (this.GlobalFilters.Count > 0)
{
filterTransform = new FilterAction(
this.GlobalFilters,
this.cancellationTokenSource.Token,
this.pipelineConfiguration.MaxEventBatchSize,
this.pipelineConfiguration.MaxConcurrency,
healthReporter);
var globalFiltersBlock = filterTransform.GetFilterBlock();
this.pipelineDisposables.Add(batcher.LinkTo(globalFiltersBlock, propagateCompletion));
sinkSource = globalFiltersBlock;
}
else
{
sinkSource = batcher;
}
if (sinks.Count > 1)
{
var broadcaster = new BroadcastBlock <EventData[]>(
(events) => events?.Select((e) => e.DeepClone()).ToArray(),
new DataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.MaxEventBatchSize,
CancellationToken = this.cancellationTokenSource.Token
});
this.pipelineDisposables.Add(sinkSource.LinkTo(broadcaster, propagateCompletion));
sinkSource = broadcaster;
}
foreach (var sink in sinks)
{
ISourceBlock <EventData[]> outputSource = sinkSource;
if (sink.Filters != null && sink.Filters.Count > 0)
{
filterTransform = new FilterAction(
sink.Filters,
this.cancellationTokenSource.Token,
this.pipelineConfiguration.MaxEventBatchSize,
this.pipelineConfiguration.MaxConcurrency,
healthReporter);
var filterBlock = filterTransform.GetFilterBlock();
this.pipelineDisposables.Add(sinkSource.LinkTo(filterBlock, propagateCompletion));
outputSource = filterBlock;
}
OutputAction outputAction = new OutputAction(
sink.Output,
this.cancellationTokenSource.Token,
this.pipelineConfiguration.MaxEventBatchSize,
this.pipelineConfiguration.MaxConcurrency,
healthReporter);
ActionBlock <EventData[]> outputBlock = outputAction.GetOutputBlock();
this.pipelineDisposables.Add(outputSource.LinkTo(outputBlock, propagateCompletion));
}
IObserver <EventData> inputBufferObserver = new TargetBlockObserver <EventData>(inputBuffer, this.HealthReporter);
this.inputSubscriptions = new List <IDisposable>(inputs.Count);
foreach (var input in inputs)
{
this.inputSubscriptions.Add(input.Subscribe(inputBufferObserver));
}
this.disposed = false;
this.disposeDependencies = disposeDependencies;
}
private void Initialize(IReadOnlyCollection <EtwProviderConfiguration> providers, IHealthReporter healthReporter)
{
this.healthReporter = healthReporter;
this.subject = new EventFlowSubject <EventData>();
this.isDisposed = false;
this.SessionFactory = () => new StandardTraceEventSession(healthReporter);
this.Providers = providers;
// The session is not started until Activate() is called.
}
// Test constructor
internal StdOutput(IHealthReporter healthReporter, Action <string> outputWriter) : this(healthReporter)
{
Requires.NotNull(outputWriter, nameof(outputWriter));
this.writeOutput = outputWriter;
}
public PerformanceCounterInput(PerformanceCounterInputConfiguration configuration, IHealthReporter healthReporter)
{
Requires.NotNull(configuration, nameof(configuration));
Requires.NotNull(healthReporter, nameof(healthReporter));
Initialize(configuration, healthReporter);
}
public static EventData ToEventData(this EventWrittenEventArgs eventSourceEvent, IHealthReporter healthReporter, string context)
{
Debug.Assert(healthReporter != null);
EventData eventData = new EventData
{
ProviderName = eventSourceEvent.EventSource.Name,
Timestamp = DateTime.UtcNow,
Level = (LogLevel)(int)eventSourceEvent.Level,
Keywords = (long)eventSourceEvent.Keywords
};
eventSourceEvent.ExtractPayloadData(eventData);
eventData.AddPayloadProperty("EventId", eventSourceEvent.EventId, healthReporter, context);
eventData.AddPayloadProperty("EventName", eventSourceEvent.EventName, healthReporter, context);
eventData.AddPayloadProperty("ActivityID", ActivityPathString(eventSourceEvent.ActivityId), healthReporter, context);
try
{
if (eventSourceEvent.Message != null)
{
// If the event has a badly formatted manifest, the FormattedMessage property getter might throw
eventData.AddPayloadProperty("Message",
string.Format(CultureInfo.InvariantCulture, eventSourceEvent.Message, eventSourceEvent.Payload.ToArray()),
healthReporter,
context);
}
}
catch { }
return(eventData);
}
private static void ExtractEventCounterPayloadData(this EventWrittenEventArgs eventSourceEvent, EventData eventData, IHealthReporter healthReporter, string context)
{
foreach (var payload in (IDictionary <string, object>)eventSourceEvent.Payload[0])
{
eventData.AddPayloadProperty(payload.Key, payload.Value, healthReporter, context);
eventData.SetMetadata(CreateMetricMetadata(payload.Key));
}
}
public TableStorageEventListener(IConfigurationProvider configurationProvider, IHealthReporter healthReporter)
: base(configurationProvider, healthReporter)
{
if (this.Disabled)
{
return;
}
Debug.Assert(configurationProvider != null);
this.CreateTableClient(configurationProvider);
Random randomNumberGenerator = new Random();
this.instanceId = randomNumberGenerator.Next(100000000).ToString("D8");
this.nextEntityId = 0;
this.identityIdResetLock = new object();
this.Sender = new ConcurrentEventSender <EventData>(
eventBufferSize: 1000,
maxConcurrency: MaxConcurrentPartitions,
batchSize: 50,
noEventsDelay: TimeSpan.FromMilliseconds(1000),
transmitterProc: this.SendEventsAsync,
healthReporter: healthReporter);
}
public ElasticSearchOutput(ElasticSearchOutputConfiguration elasticSearchOutputConfiguration, IHealthReporter healthReporter)
{
Requires.NotNull(elasticSearchOutputConfiguration, nameof(elasticSearchOutputConfiguration));
Requires.NotNull(healthReporter, nameof(healthReporter));
this.healthReporter = healthReporter;
// Clone the configuration instance since we are going to hold onto it (via this.connectionData)
Initialize(elasticSearchOutputConfiguration.DeepClone());
}
public UnitTestFilter CreateItem(IConfiguration configuration, IHealthReporter healthReporter)
{
return(new UnitTestFilter());
}