private void LoadBackends(StatsdnetConfiguration config, ISystemMetricsService systemMetrics)
{
foreach (var backend in config.GetConfiguredBackends(systemMetrics))
{
AddBackend(backend, systemMetrics, backend.Name);
}
}
private void AddAggregator(MessageType targetType,
ActionBlock <StatsdMessage> aggregator,
ISystemMetricsService systemMetrics)
{
_router.AddTarget(targetType, aggregator);
systemMetrics.LogCount("startup.aggregator." + targetType.ToString());
}
public TcpStatsListener(int port, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
IsListening = false;
_activeConnections = 0;
_tcpListener = new TcpListener(IPAddress.Any, port);
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_log = SuperCheapIOC.Resolve<ILog>();
_systemMetrics = systemMetrics;
var config = new SqlServerConfiguration(configElement.Attribute("connectionString").Value, configElement.ToInt("writeBatchSize"));
_connectionString = config.ConnectionString;
_collectorName = collectorName;
_retries = config.Retries;
InitialiseRetryHandling();
_batchBlock = new BatchBlock<GraphiteLine>(config.WriteBatchSize);
_actionBlock = new ActionBlock<GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 });
_batchBlock.LinkTo(_actionBlock);
_batchBlock.Completion.ContinueWith(p => _actionBlock.Complete());
_actionBlock.Completion.ContinueWith(p => { _isActive = false; });
_completionTask = new Task(() =>
{
_log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete.");
_batchBlock.Complete();
_actionBlock.Completion.Wait();
});
}
private void LoadListeners(StatsdnetConfiguration config, ISystemMetricsService systemMetrics)
{
// Load listeners - done last and once the rest of the chain is in place
foreach (var listenerConfig in config.Listeners)
{
if (listenerConfig is UDPListenerConfiguration)
{
var udpConfig = listenerConfig as UDPListenerConfiguration;
AddListener(new UdpStatsListener(udpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.udp." + udpConfig.Port);
}
else if (listenerConfig is TCPListenerConfiguration)
{
var tcpConfig = listenerConfig as TCPListenerConfiguration;
AddListener(new TcpStatsListener(tcpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.tcp." + tcpConfig.Port);
}
else if (listenerConfig is HTTPListenerConfiguration)
{
var httpConfig = listenerConfig as HTTPListenerConfiguration;
AddListener(new HttpStatsListener(httpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.http." + httpConfig.Port);
}
else if (listenerConfig is StatsdnetListenerConfiguration)
{
var statsdnetConfig = listenerConfig as StatsdnetListenerConfiguration;
AddListener(new StatsdnetTcpListener(statsdnetConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.statsdnet." + statsdnetConfig.Port);
}
}
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
var config = new SqlServerConfiguration(configElement.Attribute("connectionString").Value,
configElement.ToInt("writeBatchSize"));
_connectionString = config.ConnectionString;
_collectorName = collectorName;
_retries = config.Retries;
InitialiseRetryHandling();
_batchBlock = new BatchBlock <GraphiteLine>(config.WriteBatchSize);
_actionBlock = new ActionBlock <GraphiteLine[]>(p => SendToDB(p),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1
});
_batchBlock.LinkTo(_actionBlock);
_batchBlock.Completion.ContinueWith(p => _actionBlock.Complete());
_actionBlock.Completion.ContinueWith(p => { IsActive = false; });
Completion = new Task(() =>
{
_log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete.");
_batchBlock.Complete();
_actionBlock.Completion.Wait();
});
}
public TcpStatsListener(int port, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
IsListening = false;
_activeConnections = 0;
_tcpListener = new TcpListener(IPAddress.Any, port);
}
public MSSQLRelayListener(string connectionString,
TimeSpan pollInterval,
CancellationToken cancellationToken,
int batchSize,
bool deleteAfterSend,
ISystemMetricsService metrics)
{
_connectionString = connectionString;
_intervalService = new IntervalService(pollInterval, cancellationToken);
_cancellationToken = cancellationToken;
_batchSize = batchSize;
_deleteAfterSend = deleteAfterSend;
_metrics = metrics;
var stopwatch = new Stopwatch();
_intervalService.Elapsed += (sender, e) =>
{
if (IsListening)
{
_intervalService.Cancel(true);
stopwatch.Restart();
ReadAndFeed();
stopwatch.Stop();
metrics.LogCount("listeners.mssql-relay.feedTimeSeconds", Convert.ToInt32(stopwatch.Elapsed.TotalSeconds));
// Only continue the interval service if cancellation
// isn't in progress
if (!cancellationToken.IsCancellationRequested)
{
_intervalService.Start();
}
}
};
}
public MSSQLRelayListener(string connectionString,
TimeSpan pollInterval,
CancellationToken cancellationToken,
int batchSize,
bool deleteAfterSend,
ISystemMetricsService metrics)
{
_connectionString = connectionString;
_intervalService = new IntervalService(pollInterval, cancellationToken);
_cancellationToken = cancellationToken;
_batchSize = batchSize;
_deleteAfterSend = deleteAfterSend;
_metrics = metrics;
var stopwatch = new Stopwatch();
_intervalService.Elapsed += (sender, e) =>
{
if (IsListening)
{
_intervalService.Cancel(true);
stopwatch.Restart();
ReadAndFeed();
stopwatch.Stop();
metrics.LogCount("listeners.mssql-relay.feedTimeSeconds", Convert.ToInt32(stopwatch.Elapsed.TotalSeconds));
// Only continue the interval service if cancellation
// isn't in progress
if (!cancellationToken.IsCancellationRequested)
{
_intervalService.Start();
}
}
};
}
public SqlServerBackend(string connectionString,
string collectorName,
ISystemMetricsService systemMetrics,
int retries = 3,
int batchSize = 50)
{
_log = SuperCheapIOC.Resolve <ILog>();
_connectionString = connectionString;
_collectorName = collectorName;
_systemMetrics = systemMetrics;
_retries = retries;
InitialiseRetryHandling();
_batchBlock = new BatchBlock <GraphiteLine>(batchSize);
_actionBlock = new ActionBlock <GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
});
_batchBlock.LinkTo(_actionBlock);
_batchBlock.Completion.ContinueWith(p => _actionBlock.Complete());
_actionBlock.Completion.ContinueWith(p => { _isActive = false; });
_completionTask = new Task(() =>
{
_log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete.");
_batchBlock.Complete();
_actionBlock.Completion.Wait();
});
}
public UDPRawStatsSender(string host, int port, ISystemMetricsService systemMetrics)
{
var ipv4Address = Dns.GetHostAddresses(host).First(p => p.AddressFamily == AddressFamily.InterNetwork);
_client = new UdpClient();
_client.Connect(ipv4Address, port);
_systemMetrics = systemMetrics;
}
public HttpStatsListener(int port, ISystemMetricsService systemMetrics)
{
_listener = new HttpListener();
_listener.Prefixes.Add("http://*:" + port + "/");
_listener.AuthenticationSchemes = AuthenticationSchemes.Anonymous;
_systemMetrics = systemMetrics;
IsListening = false;
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_isActive = true;
_completionTask = new Task(() => { _isActive = false; });
var fileNameAttribute = configElement.Attribute("filename");
_outputFileName = (fileNameAttribute != null) ? fileNameAttribute.Value : "FileBackend.txt";
}
public StatsdnetForwardingClient(string host, int port, ISystemMetricsService systemMetrics, int numRetries = 1, bool enableCompression = true)
{
_host = host;
_port = port;
_systemMetrics = systemMetrics;
_numRetries = numRetries;
_enableCompression = enableCompression;
_client = new TcpClient();
_log = SuperCheapIOC.Resolve <ILog>();
}
public GraphiteBackend(string host, int port, ISystemMetricsService systemMetrics)
{
_log = SuperCheapIOC.Resolve <ILog>();
var ipAddress = Utility.HostToIPv4Address(host);
_client = new UdpClient();
_client.Connect(ipAddress, port);
_systemMetrics = systemMetrics;
_completionTask = new Task(() => { _isActive = false; });
}
public StatsdnetForwardingClient(string host, int port, ISystemMetricsService systemMetrics, int numRetries = 1, bool enableCompression = true)
{
_host = host;
_port = port;
_systemMetrics = systemMetrics;
_numRetries = numRetries;
_enableCompression = enableCompression;
_client = new TcpClient();
_log = SuperCheapIOC.Resolve<ILog>();
}
public StatsdnetTcpListener(int port, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
IsListening = false;
_activeConnections = 0;
_tcpListener = new TcpListener(IPAddress.Any, port);
_decoderBlock = new ActionBlock <DecoderBlockPacket>((data) => { DecodePacketAndForward(data); },
Utility.UnboundedExecution());
}
public StatsdnetTcpListener(int port, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
_log = SuperCheapIOC.Resolve<ILog>();
IsListening = false;
_activeConnections = 0;
_tcpListener = new TcpListener(IPAddress.Any, port);
_decoderBlock = new ActionBlock<DecoderBlockPacket>((data) => { DecodePacketAndForward(data); },
Utility.UnboundedExecution());
}
public void Setup()
{
_statsd = new Statsd();
_listener = new InAppListener();
_backend = new InAppBackend();
_intervalService = new ControllableIntervalService();
_outputBlock = new OutputBufferBlock <GraphiteLine>();
_client = new StatsdClient.Statsd("", 0, outputChannel: new InAppListenerOutputChannel(_listener));
_statsd.AddListener(_listener);
_statsd.AddBackend(_backend, _systemMetrics, "testing");
_systemMetrics = Substitute.For <ISystemMetricsService>();
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
Completion = new Task(() => { IsActive = false; });
_batchBlock = new BatchBlock <GraphiteLine>(BATCH_SIZE);
_senderBlock = new ActionBlock <GraphiteLine[]>(batch => SendBatch(batch), Utility.OneAtATimeExecution());
_batchBlock.LinkTo(_senderBlock);
IsActive = true;
_host = configElement.Attribute("host").Value;
_port = configElement.ToInt("port");
}
public void Setup()
{
SuperCheapIOC.Reset();
_statsd = new Statsd();
_listener = new InAppListener();
_backend = new InAppBackend();
_intervalService = new ControllableIntervalService();
_outputBlock = new OutputBufferBlock<GraphiteLine>();
_client = new StatsdClient.Statsd("", 0, outputChannel : new InAppListenerOutputChannel(_listener));
_statsd.AddListener(_listener);
_statsd.AddBackend(_backend, _systemMetrics, "testing");
_systemMetrics = new Mock<ISystemMetricsService>().Object;
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_log = SuperCheapIOC.Resolve<ILog>();
_systemMetrics = systemMetrics;
_completionTask = new Task(() => { _isActive = false; });
_senderBlock = new ActionBlock<GraphiteLine>((message) => SendLine(message), Utility.UnboundedExecution());
_isActive = true;
var config = new GraphiteConfiguration(configElement.Attribute("host").Value, configElement.ToInt("port"));
var ipAddress = Utility.HostToIPv4Address(config.Host);
_client = new UdpClient();
_client.Connect(ipAddress, config.Port);
}
public void AddBackend(IBackend backend, ISystemMetricsService systemMetrics, string name)
{
_log.InfoFormat("Adding backend {0} named '{1}'", backend.GetType().Name, name);
_backends.Add(backend);
_messageBroadcaster.LinkTo(backend);
backend.Completion.LogAndContinueWith(name, () =>
{
if (_backends.All(q => !q.IsActive))
{
_shutdownComplete.Set();
}
});
systemMetrics.LogCount("startup.backend." + name);
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
Completion = new Task(() => { IsActive = false; });
_senderBlock = new ActionBlock <GraphiteLine>(message => SendLine(message), Utility.OneAtATimeExecution());
IsActive = true;
var config = new GraphiteConfiguration(configElement.Attribute("host").Value, configElement.ToInt("port"));
var ipAddress = Utility.HostToIPv4Address(config.Host);
_client = new UdpClient();
_client.Connect(ipAddress, config.Port);
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_log = SuperCheapIOC.Resolve <ILog>();
_systemMetrics = systemMetrics;
_completionTask = new Task(() => { _isActive = false; });
_senderBlock = new ActionBlock <GraphiteLine>((message) => SendLine(message), Utility.UnboundedExecution());
_isActive = true;
var config = new GraphiteConfiguration(configElement.Attribute("host").Value, configElement.ToInt("port"));
var ipAddress = Utility.HostToIPv4Address(config.Host);
_client = new UdpClient();
_client.Connect(ipAddress, config.Port);
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
var config = new StatsdBackendConfiguration(configElement.Attribute("host").Value,
configElement.ToInt("port"),
Utility.ConvertToTimespan(configElement.Attribute("flushInterval").Value),
configElement.ToBoolean("enableCompression", true));
_client = new StatsdnetForwardingClient(config.Host, config.Port, _systemMetrics);
_bufferBlock = new TimedBufferBlock <GraphiteLine[]>(config.FlushInterval, PostMetrics);
Completion = new Task(() => { IsActive = false; });
IsActive = true;
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_completionTask = new Task(() => IsActive = false);
_log = SuperCheapIOC.Resolve <ILog>();
_config = parseConfig(configElement);
_reporter = new SignalFxReporter(_config.BaseURI, _config.ApiToken, _config.PostTimeout);
_processBlock = new ActionBlock <Bucket>(bucket => ProcessBucket(bucket), Utility.UnboundedExecution());
_batchBlock = new BatchBlock <TypeDatapoint>(_config.MaxBatchSize);
_outputBlock = new ActionBlock <TypeDatapoint[]>(datapoint => ProcessDatapoints(datapoint), Utility.OneAtATimeExecution());
_batchBlock.LinkTo(_outputBlock);
_triggerBatchTimer = new Timer((state) => trigger(state), null, TimeSpan.FromSeconds(1), _config.MaxTimeBetweenBatches);
IsActive = true;
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
var config = new StatsdBackendConfiguration(configElement.Attribute("host").Value,
configElement.ToInt("port"),
Utility.ConvertToTimespan(configElement.Attribute("flushInterval").Value),
configElement.ToBoolean("enableCompression", true));
_client = new StatsdnetForwardingClient(config.Host, config.Port, _systemMetrics);
_bufferBlock = new TimedBufferBlock<GraphiteLine[]>(config.FlushInterval, PostMetrics);
_completionTask = new Task(() =>
{
_isActive = false;
});
_isActive = true;
}
private void LoadListeners(StatsdnetConfiguration config,
CancellationToken cancellationToken,
ISystemMetricsService systemMetrics)
{
// Load listeners - done last and once the rest of the chain is in place
foreach (var listenerConfig in config.Listeners)
{
if (listenerConfig is UDPListenerConfiguration)
{
var udpConfig = listenerConfig as UDPListenerConfiguration;
AddListener(new UdpStatsListener(udpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.udp." + udpConfig.Port);
}
else if (listenerConfig is TCPListenerConfiguration)
{
var tcpConfig = listenerConfig as TCPListenerConfiguration;
AddListener(new TcpStatsListener(tcpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.tcp." + tcpConfig.Port);
}
else if (listenerConfig is HTTPListenerConfiguration)
{
var httpConfig = listenerConfig as HTTPListenerConfiguration;
AddListener(new HttpStatsListener(httpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.http." + httpConfig.Port);
}
else if (listenerConfig is StatsdnetListenerConfiguration)
{
var statsdnetConfig = listenerConfig as StatsdnetListenerConfiguration;
AddListener(new StatsdnetTcpListener(statsdnetConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.statsdnet." + statsdnetConfig.Port);
}
else if (listenerConfig is MSSQLRelayListenerConfiguration)
{
var mssqlRelayConfig = listenerConfig as MSSQLRelayListenerConfiguration;
AddListener(new MSSQLRelayListener(mssqlRelayConfig.ConnectionString,
mssqlRelayConfig.PollInterval,
cancellationToken,
mssqlRelayConfig.BatchSize,
mssqlRelayConfig.DeleteAfterSend,
systemMetrics));
}
}
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_completionTask = new Task(() => IsActive = false);
_systemMetrics = systemMetrics;
var config = new LibratoBackendConfiguration(
email: configElement.Attribute("email").Value,
token: configElement.Attribute("token").Value,
numRetries: configElement.ToInt("numRetries"),
retryDelay: Utility.ConvertToTimespan(configElement.Attribute("retryDelay").Value),
postTimeout: Utility.ConvertToTimespan(configElement.Attribute("postTimeout").Value),
maxBatchSize: configElement.ToInt("maxBatchSize"),
countersAsGauges: configElement.ToBoolean("countersAsGauges")
);
_config = config;
_source = collectorName;
_serviceVersion = Assembly.GetEntryAssembly().GetName().Version.ToString();
_preprocessorBlock = new ActionBlock <Bucket>(bucket => ProcessBucket(bucket), Utility.UnboundedExecution());
_batchBlock = new BatchBlock <LibratoMetric>(_config.MaxBatchSize);
_outputBlock = new ActionBlock <LibratoMetric[]>(lines => PostToLibrato(lines), Utility.OneAtATimeExecution());
_batchBlock.LinkTo(_outputBlock);
_client = new HttpClient()
{
BaseAddress = new Uri(LIBRATO_API_URL)
};
var authByteArray = Encoding.ASCII.GetBytes(string.Format("{0}:{1}", _config.Email, _config.Token));
_client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(authByteArray));
_client.Timeout = TimeSpan.FromMilliseconds(_config.PostTimeout.TotalMilliseconds);
_retryPolicy = Policy.Handle <TimeoutException>().WaitAndRetry(_config.NumRetries, retryAttempt => _config.RetryDelay, (exception, timeSpan) =>
{
_log.WarnException(String.Format("Retry failed. Trying again. Delay {1}, Error: {2}", timeSpan, exception.Message), exception);
_systemMetrics.LogCount("backends.librato.retry");
});
IsActive = true;
}
public UdpStatsListener(int port, ISystemMetricsService systemMetrics)
{
_port = port;
_systemMetrics = systemMetrics;
_preprocessorBlock = new ActionBlock <byte []>((data) =>
{
// Log a metric to see what the difference between the read buffer and the max buffer size is
_systemMetrics.LogGauge("listeners.udp.buffer.max", MAX_BUFFER_SIZE);
_systemMetrics.LogGauge("listeners.udp.buffer.current", data.Length);
_systemMetrics.LogCount("listeners.udp.bytes", data.Length);
string rawPacket = Encoding.UTF8.GetString(data);
string [] lines = rawPacket.Replace("\r", "").Split(new char [] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
for (int index = 0; index < lines.Length; index++)
{
_targetBlock.Post(lines [index]);
}
_systemMetrics.LogCount("listeners.udp.lines", lines.Length);
}, Utility.UnboundedExecution());
}
public UdpStatsListener(int port, ISystemMetricsService systemMetrics)
{
_port = port;
_systemMetrics = systemMetrics;
_preprocessorBlock = new ActionBlock<byte []>( (data) =>
{
// Log a metric to see what the difference between the read buffer and the max buffer size is
_systemMetrics.LogGauge("listeners.udp.buffer.max", MAX_BUFFER_SIZE);
_systemMetrics.LogGauge("listeners.udp.buffer.current", data.Length);
_systemMetrics.LogCount( "listeners.udp.bytes", data.Length );
string rawPacket = Encoding.UTF8.GetString( data );
string [] lines = rawPacket.Replace( "\r", "" ).Split( new char [] { '\n' }, StringSplitOptions.RemoveEmptyEntries );
for ( int index = 0; index < lines.Length; index++ )
{
_targetBlock.Post( lines [ index ] );
}
_systemMetrics.LogCount( "listeners.udp.lines", lines.Length );
}, Utility.UnboundedExecution() );
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_completionTask = new Task(() => IsActive = false);
_log = SuperCheapIOC.Resolve<ILog>();
_systemMetrics = systemMetrics;
var config = new LibratoBackendConfiguration(
email: configElement.Attribute("email").Value,
token: configElement.Attribute("token").Value,
numRetries: configElement.ToInt("numRetries"),
retryDelay: Utility.ConvertToTimespan(configElement.Attribute("retryDelay").Value),
postTimeout: Utility.ConvertToTimespan(configElement.Attribute("postTimeout").Value),
maxBatchSize: configElement.ToInt("maxBatchSize"),
countersAsGauges: configElement.ToBoolean("countersAsGauges")
);
_config = config;
_source = collectorName;
_serviceVersion = Assembly.GetEntryAssembly().GetName().Version.ToString();
_preprocessorBlock = new ActionBlock<Bucket>(bucket => ProcessBucket(bucket), Utility.UnboundedExecution());
_batchBlock = new BatchBlock<LibratoMetric>(_config.MaxBatchSize);
_outputBlock = new ActionBlock<LibratoMetric[]>(lines => PostToLibrato(lines), Utility.OneAtATimeExecution());
_batchBlock.LinkTo(_outputBlock);
_client = new RestClient(LIBRATO_API_URL);
_client.Authenticator = new HttpBasicAuthenticator(_config.Email, _config.Token);
_client.Timeout = (int)_config.PostTimeout.TotalMilliseconds;
_retryPolicy = new RetryPolicy<LibratoErrorDetectionStrategy>(_config.NumRetries);
_retryPolicy.Retrying += (sender, args) =>
{
_log.Warn(String.Format("Retry {0} failed. Trying again. Delay {1}, Error: {2}", args.CurrentRetryCount, args.Delay, args.LastException.Message), args.LastException);
_systemMetrics.LogCount("backends.librato.retry");
};
_retryStrategy = new Incremental(_config.NumRetries, _config.RetryDelay, TimeSpan.FromSeconds(2));
IsActive = true;
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_completionTask = new Task(() => IsActive = false);
_log = SuperCheapIOC.Resolve <ILog>();
_systemMetrics = systemMetrics;
var config = new LibratoBackendConfiguration(
email: configElement.Attribute("email").Value,
token: configElement.Attribute("token").Value,
numRetries: configElement.ToInt("numRetries"),
retryDelay: Utility.ConvertToTimespan(configElement.Attribute("retryDelay").Value),
postTimeout: Utility.ConvertToTimespan(configElement.Attribute("postTimeout").Value),
maxBatchSize: configElement.ToInt("maxBatchSize"),
countersAsGauges: configElement.ToBoolean("countersAsGauges")
);
_config = config;
_source = collectorName;
_serviceVersion = Assembly.GetEntryAssembly().GetName().Version.ToString();
_preprocessorBlock = new ActionBlock <Bucket>(bucket => ProcessBucket(bucket), Utility.UnboundedExecution());
_batchBlock = new BatchBlock <LibratoMetric>(_config.MaxBatchSize);
_outputBlock = new ActionBlock <LibratoMetric[]>(lines => PostToLibrato(lines), Utility.OneAtATimeExecution());
_batchBlock.LinkTo(_outputBlock);
_client = new RestClient(LIBRATO_API_URL);
_client.Authenticator = new HttpBasicAuthenticator(_config.Email, _config.Token);
_client.Timeout = (int)_config.PostTimeout.TotalMilliseconds;
_retryPolicy = new RetryPolicy <LibratoErrorDetectionStrategy>(_config.NumRetries);
_retryPolicy.Retrying += (sender, args) =>
{
_log.Warn(String.Format("Retry {0} failed. Trying again. Delay {1}, Error: {2}", args.CurrentRetryCount, args.Delay, args.LastException.Message), args.LastException);
_systemMetrics.LogCount("backends.librato.retry");
};
_retryStrategy = new Incremental(_config.NumRetries, _config.RetryDelay, TimeSpan.FromSeconds(2));
IsActive = true;
}
public static TransformBlock<String, StatsdMessage> CreateMessageParserBlock(CancellationToken cancellationToken,
ISystemMetricsService systemMetrics,
ILog log)
{
var block = new TransformBlock<String, StatsdMessage>(
(line) =>
{
systemMetrics.LogCount("parser.linesSeen");
StatsdMessage message = StatsdMessageFactory.ParseMessage(line);
if (message is InvalidMessage)
{
systemMetrics.LogCount("parser.badLinesSeen");
log.Info("Bad message: " + ((InvalidMessage)message).Reason);
}
return message;
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = ExecutionDataflowBlockOptions.Unbounded,
CancellationToken = cancellationToken
});
return block;
}
public IEnumerable <IBackend> GetConfiguredBackends(ISystemMetricsService systemMetrics)
{
if (_log.IsInfoEnabled)
{
var availableBackendsString = String.Join(", ", AvailableBackends.Select(x => x.Name));
_log.InfoFormat("Available Backends: {0}", availableBackendsString);
}
foreach (var pair in BackendConfigurations)
{
string backendName = pair.Key;
IBackend backend = AvailableBackends.FirstOrDefault(x => x.Name.Equals(backendName, StringComparison.OrdinalIgnoreCase));
if (backend == null)
{
_log.WarnFormat("Unrecognized backend configuration for \"{0}\". Backend will be ignored.", backendName);
continue;
}
backend.Configure(Name, pair.Value, systemMetrics);
yield return(backend);
}
}
public IEnumerable<IBackend> GetConfiguredBackends(ISystemMetricsService systemMetrics)
{
if (_log.IsInfoEnabled)
{
var availableBackendsString = String.Join(", ", AvailableBackends.Select(x => x.Name));
_log.InfoFormat("Available Backends: {0}", availableBackendsString);
}
foreach (var pair in BackendConfigurations)
{
string backendName = pair.Key;
IBackend backend = AvailableBackends.FirstOrDefault(x => x.Name.Equals(backendName, StringComparison.OrdinalIgnoreCase));
if (backend == null)
{
_log.WarnFormat("Unrecognized backend configuration for \"{0}\". Backend will be ignored.", backendName);
continue;
}
backend.Configure(Name, pair.Value, systemMetrics);
yield return backend;
}
}
public HttpStatsListener(int port, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
_port = port;
}
private void LoadAggregators(StatsdnetConfiguration config,
IntervalService intervalService,
BroadcastBlock<Bucket> messageBroadcaster,
ISystemMetricsService systemMetrics)
{
foreach (var aggregator in config.Aggregators)
{
switch (aggregator.Key)
{
case "counters":
var counter = aggregator.Value as CounterAggregationConfig;
AddAggregator(MessageType.Counter,
TimedCounterAggregatorBlockFactory.CreateBlock(messageBroadcaster,
counter.Namespace,
intervalService,
_log),
systemMetrics);
break;
case "gauges":
var gauge = aggregator.Value as GaugeAggregatorConfig;
AddAggregator(MessageType.Gauge,
TimedGaugeAggregatorBlockFactory.CreateBlock(messageBroadcaster,
gauge.Namespace,
gauge.RemoveZeroGauges,
intervalService,
_log),
systemMetrics);
break;
case "calendargrams":
var calendargram = aggregator.Value as CalendargramAggregationConfig;
AddAggregator(MessageType.Calendargram,
TimedCalendargramAggregatorBlockFactory.CreateBlock(messageBroadcaster,
calendargram.Namespace,
intervalService,
new TimeWindowService(),
_log),
systemMetrics);
break;
case "timers":
var timer = aggregator.Value as TimersAggregationConfig;
AddAggregator(MessageType.Timing,
TimedLatencyAggregatorBlockFactory.CreateBlock(messageBroadcaster,
timer.Namespace,
intervalService,
timer.CalculateSumSquares,
_log),
systemMetrics);
// Add Percentiles
foreach (var percentile in timer.Percentiles)
{
AddAggregator(MessageType.Timing,
TimedLatencyPercentileAggregatorBlockFactory.CreateBlock(messageBroadcaster,
timer.Namespace,
intervalService,
percentile.Threshold,
percentile.Name,
_log),
systemMetrics);
}
break;
}
}
// Add the Raw (pass-through) aggregator
AddAggregator(MessageType.Raw,
PassThroughBlockFactory.CreateBlock(messageBroadcaster, intervalService),
systemMetrics);
}
private void LoadAggregators(StatsdnetConfiguration config,
IntervalService intervalService,
BroadcastBlock <Bucket> messageBroadcaster,
ISystemMetricsService systemMetrics)
{
foreach (var aggregator in config.Aggregators)
{
switch (aggregator.Key)
{
case "counters":
var counter = aggregator.Value as CounterAggregationConfig;
AddAggregator(MessageType.Counter,
TimedCounterAggregatorBlockFactory.CreateBlock(messageBroadcaster,
counter.Namespace,
intervalService),
systemMetrics);
break;
case "gauges":
var gauge = aggregator.Value as GaugeAggregatorConfig;
AddAggregator(MessageType.Gauge,
TimedGaugeAggregatorBlockFactory.CreateBlock(messageBroadcaster,
gauge.Namespace,
gauge.RemoveZeroGauges,
intervalService),
systemMetrics);
break;
case "calendargrams":
var calendargram = aggregator.Value as CalendargramAggregationConfig;
AddAggregator(MessageType.Calendargram,
TimedCalendargramAggregatorBlockFactory.CreateBlock(messageBroadcaster,
calendargram.Namespace,
intervalService,
new TimeWindowService()),
systemMetrics);
break;
case "timers":
var timer = aggregator.Value as TimersAggregationConfig;
AddAggregator(MessageType.Timing,
TimedLatencyAggregatorBlockFactory.CreateBlock(messageBroadcaster,
timer.Namespace,
intervalService,
timer.CalculateSumSquares),
systemMetrics);
// Add Percentiles
foreach (var percentile in timer.Percentiles)
{
AddAggregator(MessageType.Timing,
TimedLatencyPercentileAggregatorBlockFactory.CreateBlock(messageBroadcaster,
timer.Namespace,
intervalService,
percentile.Threshold,
percentile.Name),
systemMetrics);
}
break;
}
}
// Add the Raw (pass-through) aggregator
AddAggregator(MessageType.Raw,
PassThroughBlockFactory.CreateBlock(messageBroadcaster, intervalService),
systemMetrics);
}
public HttpStatsListener(int port, ISystemMetricsService systemMetrics, ICorsValidationProvider corsValidator)
{
_systemMetrics = systemMetrics;
_port = port;
_corsValidator = corsValidator;
}
public void AddBackend(IBackend backend, ISystemMetricsService systemMetrics, string name)
{
_log.InfoFormat("Adding backend {0} named '{1}'", backend.GetType().Name, name);
_backends.Add(backend);
_messageBroadcaster.LinkTo(backend);
backend.Completion.LogAndContinueWith(_log, name, () =>
{
if (_backends.All(q => !q.IsActive))
{
_shutdownComplete.Set();
}
});
systemMetrics.LogCount("startup.backend." + name);
}
public HttpStatsListener(int port, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
_port = port;
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_isActive = true;
_completionTask = new Task(() => { _isActive = false; });
}
public static TransformBlock <String, StatsdMessage> CreateMessageParserBlock(CancellationToken cancellationToken,
ISystemMetricsService systemMetrics,
ILog log)
{
var block = new TransformBlock <String, StatsdMessage>(
(line) =>
{
systemMetrics.LogCount("parser.linesSeen");
StatsdMessage message = StatsdMessageFactory.ParseMessage(line);
if (message is InvalidMessage)
{
systemMetrics.LogCount("parser.badLinesSeen");
log.Info("Bad message: " + ((InvalidMessage)message).Reason);
}
return(message);
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = ExecutionDataflowBlockOptions.Unbounded,
CancellationToken = cancellationToken
});
return(block);
}
private void LoadListeners(StatsdnetConfiguration config,
CancellationToken cancellationToken,
ISystemMetricsService systemMetrics)
{
// Load listeners - done last and once the rest of the chain is in place
foreach (var listenerConfig in config.Listeners)
{
if (listenerConfig is UDPListenerConfiguration)
{
var udpConfig = listenerConfig as UDPListenerConfiguration;
AddListener(new UdpStatsListener(udpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.udp." + udpConfig.Port);
}
else if (listenerConfig is TCPListenerConfiguration)
{
var tcpConfig = listenerConfig as TCPListenerConfiguration;
AddListener(new TcpStatsListener(tcpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.tcp." + tcpConfig.Port);
}
else if (listenerConfig is HTTPListenerConfiguration)
{
var httpConfig = listenerConfig as HTTPListenerConfiguration;
AddListener(new HttpStatsListener(httpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.http." + httpConfig.Port);
}
else if (listenerConfig is StatsdnetListenerConfiguration)
{
var statsdnetConfig = listenerConfig as StatsdnetListenerConfiguration;
AddListener(new StatsdnetTcpListener(statsdnetConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.statsdnet." + statsdnetConfig.Port);
}
else if (listenerConfig is MSSQLRelayListenerConfiguration)
{
var mssqlRelayConfig = listenerConfig as MSSQLRelayListenerConfiguration;
AddListener(new MSSQLRelayListener(mssqlRelayConfig.ConnectionString,
mssqlRelayConfig.PollInterval,
cancellationToken,
mssqlRelayConfig.BatchSize,
mssqlRelayConfig.DeleteAfterSend,
systemMetrics));
}
}
}
private void LoadListeners(StatsdnetConfiguration config, ISystemMetricsService systemMetrics)
{
// Load listeners - done last and once the rest of the chain is in place
foreach (var listenerConfig in config.Listeners)
{
if (listenerConfig is UDPListenerConfiguration)
{
var udpConfig = listenerConfig as UDPListenerConfiguration;
AddListener(new UdpStatsListener(udpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.udp." + udpConfig.Port);
}
else if (listenerConfig is TCPListenerConfiguration)
{
var tcpConfig = listenerConfig as TCPListenerConfiguration;
AddListener(new TcpStatsListener(tcpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.tcp." + tcpConfig.Port);
}
else if (listenerConfig is HTTPListenerConfiguration)
{
var httpConfig = listenerConfig as HTTPListenerConfiguration;
AddListener(new HttpStatsListener(httpConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.http." + httpConfig.Port);
}
else if (listenerConfig is StatsdnetListenerConfiguration)
{
var statsdnetConfig = listenerConfig as StatsdnetListenerConfiguration;
AddListener(new StatsdnetTcpListener(statsdnetConfig.Port, systemMetrics));
systemMetrics.LogCount("startup.listener.statsdnet." + statsdnetConfig.Port);
}
}
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_isActive = true;
_completionTask = new Task(() => { _isActive = false; });
}
private void LoadBackends(StatsdnetConfiguration config, ISystemMetricsService systemMetrics)
{
foreach (var backend in config.GetConfiguredBackends(systemMetrics))
{
AddBackend(backend, systemMetrics, backend.Name);
}
}
private void AddAggregator(MessageType targetType,
ActionBlock<StatsdMessage> aggregator,
ISystemMetricsService systemMetrics)
{
_router.AddTarget(targetType, aggregator);
systemMetrics.LogCount("startup.aggregator." + targetType.ToString());
}
