public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_log = SuperCheapIOC.Resolve <ILog>();
_systemMetrics = systemMetrics;
var config = new SqlServerConfiguration(configElement.Attribute("connectionString").Value, configElement.ToInt("writeBatchSize"));
_connectionString = config.ConnectionString;
_collectorName = collectorName;
_retries = config.Retries;
InitialiseRetryHandling();
_batchBlock = new BatchBlock <GraphiteLine>(config.WriteBatchSize);
_actionBlock = new ActionBlock <GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
});
_batchBlock.LinkTo(_actionBlock);
_batchBlock.Completion.ContinueWith(p => _actionBlock.Complete());
_actionBlock.Completion.ContinueWith(p => { _isActive = false; });
_completionTask = new Task(() =>
{
_log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete.");
_batchBlock.Complete();
_actionBlock.Completion.Wait();
});
}
public SqlServerBackend(string connectionString,
string collectorName,
ISystemMetricsService systemMetrics,
int retries = 3,
int batchSize = 50)
{
_log = SuperCheapIOC.Resolve <ILog>();
_connectionString = connectionString;
_collectorName = collectorName;
_systemMetrics = systemMetrics;
_retries = retries;
InitialiseRetryHandling();
_batchBlock = new BatchBlock <GraphiteLine>(batchSize);
_actionBlock = new ActionBlock <GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
});
_batchBlock.LinkTo(_actionBlock);
_batchBlock.Completion.ContinueWith(p => _actionBlock.Complete());
_actionBlock.Completion.ContinueWith(p => { _isActive = false; });
_completionTask = new Task(() =>
{
_log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete.");
_batchBlock.Complete();
_actionBlock.Completion.Wait();
});
}
public Statsd(StatsdnetConfiguration config)
: this(config.Name)
{
_log.Info("statsd.net loading config.");
var systemMetrics = SuperCheapIOC.Resolve <ISystemMetricsService>();
systemMetrics.HideSystemStats = config.HideSystemStats;
LoadBackends(config, systemMetrics);
// Load Aggregators
var intervalServices = new List <IIntervalService>();
var intervalService = new IntervalService(config.FlushInterval,
_tokenSource.Token);
intervalServices.Add(intervalService);
LoadAggregators(config,
intervalService,
_messageBroadcaster,
systemMetrics);
// Load Listeners
LoadListeners(config, _tokenSource.Token, systemMetrics);
// Now start the interval service
intervalServices.ForEach(p => p.Start());
// Announce that we've started
systemMetrics.LogCount("started");
}
public GraphiteBackend(string host, int port, ISystemMetricsService systemMetrics)
{
_log = SuperCheapIOC.Resolve <ILog>();
var ipAddress = Utility.HostToIPv4Address(host);
_client = new UdpClient();
_client.Connect(ipAddress, port);
_systemMetrics = systemMetrics;
_completionTask = new Task(() => { _isActive = false; });
}
public StatsdnetForwardingClient(string host, int port, ISystemMetricsService systemMetrics, int numRetries = 1, bool enableCompression = true)
{
_host = host;
_port = port;
_systemMetrics = systemMetrics;
_numRetries = numRetries;
_enableCompression = enableCompression;
_client = new TcpClient();
_log = SuperCheapIOC.Resolve <ILog>();
}
public StatsdnetTcpListener(int port, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
_log = SuperCheapIOC.Resolve <ILog>();
IsListening = false;
_activeConnections = 0;
_tcpListener = new TcpListener(IPAddress.Any, port);
_decoderBlock = new ActionBlock <DecoderBlockPacket>((data) => { DecodePacketAndForward(data); },
Utility.UnboundedExecution());
}
public void Setup()
{
SuperCheapIOC.Reset();
_statsd = new Statsd();
_listener = new InAppListener();
_backend = new InAppBackend();
_intervalService = new ControllableIntervalService();
_outputBlock = new OutputBufferBlock <GraphiteLine>();
_client = new StatsdClient.Statsd("", 0, outputChannel: new InAppListenerOutputChannel(_listener));
_statsd.AddListener(_listener);
_statsd.AddBackend(_backend);
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_log = SuperCheapIOC.Resolve <ILog>();
_systemMetrics = systemMetrics;
_completionTask = new Task(() => { _isActive = false; });
_batchBlock = new BatchBlock <GraphiteLine>(BATCH_SIZE);
_senderBlock = new ActionBlock <GraphiteLine[]>((batch) => SendBatch(batch), Utility.OneAtATimeExecution());
_batchBlock.LinkTo(_senderBlock);
_isActive = true;
_host = configElement.Attribute("host").Value;
_port = configElement.ToInt("port");
}
public Statsd(string serviceName = null)
{
LoggingBootstrap.Configure();
_log.Info("statsd.net starting.");
_tokenSource = new CancellationTokenSource();
_shutdownComplete = new ManualResetEvent(false);
SuperCheapIOC.Add(_log);
var systemInfoService = new SystemInfoService();
SuperCheapIOC.Add(systemInfoService as ISystemInfoService);
serviceName = serviceName ?? systemInfoService.HostName;
var systemMetricsService = new SystemMetricsService("statsdnet", serviceName);
SuperCheapIOC.Add(systemMetricsService as ISystemMetricsService);
/**
* The flow is:
* Listeners ->
* Message Parser ->
* router ->
* Aggregator ->
* Broadcaster ->
* Backends
*/
// Initialise the core blocks
_router = new StatsdMessageRouterBlock();
_messageParser = MessageParserBlockFactory.CreateMessageParserBlock(_tokenSource.Token,
SuperCheapIOC.Resolve <ISystemMetricsService>(),
_log);
_messageParser.LinkTo(_router);
_messageParser.Completion.LogAndContinueWith(_log, "MessageParser", () =>
{
_log.Info("MessageParser: Completion signaled. Notifying the MessageBroadcaster.");
_messageBroadcaster.Complete();
});
_messageBroadcaster = new BroadcastBlock <GraphiteLine>(GraphiteLine.Clone);
_messageBroadcaster.Completion.LogAndContinueWith(_log, "MessageBroadcaster", () =>
{
_log.Info("MessageBroadcaster: Completion signaled. Notifying all backends.");
_backends.ForEach(q => q.Complete());
});
// Add the broadcaster to the IOC container
SuperCheapIOC.Add <BroadcastBlock <GraphiteLine> >(_messageBroadcaster);
systemMetricsService.SetTarget(_messageBroadcaster);
_backends = new List <IBackend>();
_listeners = new List <IListener>();
}
public void AddBackend(IBackend backend, string name = "")
{
_log.InfoFormat("Adding backend {0} named '{1}'", backend.GetType().Name, name);
_backends.Add(backend);
SuperCheapIOC.Add(backend, name);
_messageBroadcaster.LinkTo(backend);
backend.Completion.LogAndContinueWith(_log, name, () =>
{
if (_backends.All(q => !q.IsActive))
{
_shutdownComplete.Set();
}
});
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_log = SuperCheapIOC.Resolve <ILog>();
_systemMetrics = systemMetrics;
_completionTask = new Task(() => { _isActive = false; });
_senderBlock = new ActionBlock <GraphiteLine>((message) => SendLine(message), Utility.UnboundedExecution());
_isActive = true;
var config = new GraphiteConfiguration(configElement.Attribute("host").Value, configElement.ToInt("port"));
var ipAddress = Utility.HostToIPv4Address(config.Host);
_client = new UdpClient();
_client.Connect(ipAddress, config.Port);
}
public SignalFxReporter(string baseURI, string apiToken, TimeSpan timeout, IWebRequestorFactory requestor = null)
{
_log = SuperCheapIOC.Resolve <ILog>();
if (requestor == null)
{
requestor = new WebRequestorFactory()
.WithUri(baseURI + "/v2/datapoint")
.WithMethod("POST")
.WithContentType("application/x-protobuf")
.WithHeader("X-SF-TOKEN", apiToken)
.WithTimeout(timeout);
}
this._requestor = requestor;
this._apiToken = apiToken;
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_completionTask = new Task(() => IsActive = false);
_log = SuperCheapIOC.Resolve <ILog>();
_config = parseConfig(configElement);
_reporter = new SignalFxReporter(_config.BaseURI, _config.ApiToken, _config.PostTimeout);
_processBlock = new ActionBlock <Bucket>(bucket => ProcessBucket(bucket), Utility.UnboundedExecution());
_batchBlock = new BatchBlock <TypeDatapoint>(_config.MaxBatchSize);
_outputBlock = new ActionBlock <TypeDatapoint[]>(datapoint => ProcessDatapoints(datapoint), Utility.OneAtATimeExecution());
_batchBlock.LinkTo(_outputBlock);
_triggerBatchTimer = new Timer((state) => trigger(state), null, TimeSpan.FromSeconds(1), _config.MaxTimeBetweenBatches);
IsActive = true;
}
private void SendMetrics(object sender, IntervalFiredEventArgs args)
{
if (_target == null)
{
return;
}
// Get a count of metrics waiting to be sent out
var outputBufferCount = SuperCheapIOC.ResolveAll <IBackend>().Sum(p => p.OutputCount);
LogGauge("outputBuffer", outputBufferCount);
LogGauge("up", 1);
var bucket = new CounterBucket(_metrics.ToArray(), args.Epoch, _prefix);
_metrics.Clear();
if (!HideSystemStats)
{
_target.Post(bucket);
}
}
private void SendMetrics(object sender, IntervalFiredEventArgs args)
{
if (_target == null)
{
return;
}
// Get a count of metrics waiting to be sent out
var outputBufferCount = SuperCheapIOC.ResolveAll <IBackend>().Sum(p => p.OutputCount);
_target.Post(new GraphiteLine(_prefix + "outputBuffer", outputBufferCount));
_target.Post(new GraphiteLine(_prefix + "up", 1));
var pairs = _metrics.ToArray();
_metrics.Clear();
foreach (var pair in pairs)
{
_target.Post(new GraphiteLine(_prefix + pair.Key, pair.Value));
}
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_completionTask = new Task(() => IsActive = false);
_log = SuperCheapIOC.Resolve <ILog>();
_systemMetrics = systemMetrics;
var config = new LibratoBackendConfiguration(
email: configElement.Attribute("email").Value,
token: configElement.Attribute("token").Value,
numRetries: configElement.ToInt("numRetries"),
retryDelay: Utility.ConvertToTimespan(configElement.Attribute("retryDelay").Value),
postTimeout: Utility.ConvertToTimespan(configElement.Attribute("postTimeout").Value),
maxBatchSize: configElement.ToInt("maxBatchSize"),
countersAsGauges: configElement.ToBoolean("countersAsGauges")
);
_config = config;
_source = collectorName;
_serviceVersion = Assembly.GetEntryAssembly().GetName().Version.ToString();
_preprocessorBlock = new ActionBlock <Bucket>(bucket => ProcessBucket(bucket), Utility.UnboundedExecution());
_batchBlock = new BatchBlock <LibratoMetric>(_config.MaxBatchSize);
_outputBlock = new ActionBlock <LibratoMetric[]>(lines => PostToLibrato(lines), Utility.OneAtATimeExecution());
_batchBlock.LinkTo(_outputBlock);
_client = new RestClient(LIBRATO_API_URL);
_client.Authenticator = new HttpBasicAuthenticator(_config.Email, _config.Token);
_client.Timeout = (int)_config.PostTimeout.TotalMilliseconds;
_retryPolicy = new RetryPolicy <LibratoErrorDetectionStrategy>(_config.NumRetries);
_retryPolicy.Retrying += (sender, args) =>
{
_log.Warn(String.Format("Retry {0} failed. Trying again. Delay {1}, Error: {2}", args.CurrentRetryCount, args.Delay, args.LastException.Message), args.LastException);
_systemMetrics.LogCount("backends.librato.retry");
};
_retryStrategy = new Incremental(_config.NumRetries, _config.RetryDelay, TimeSpan.FromSeconds(2));
IsActive = true;
}
public Relay(dynamic config)
{
LoggingBootstrap.Configure();
_log.Info("statsdrelay is starting up.");
_tokenSource = new CancellationTokenSource();
_shutdownComplete = new ManualResetEvent(false);
_listeners = new List <IListener>();
var systemInfoService = new SystemInfoService();
var relayMetrics = new RelayMetricsService("relay", _tokenSource.Token, systemInfoService.HostName);
SuperCheapIOC.Add(relayMetrics as ISystemMetricsService);
/* Pipeline is
* UDPStatsListener
* HTTPStatsListener
* TCPStatsListener
* -> MessageParserBlock
* -> BatchBlock
* -> UDPRawStatsSender
*/
var udpSender = new UDPRawStatsSender(config.target.host, (int)config.target.port, relayMetrics);
var outputBlock = new ActionBlock <StatsdMessage[]>((lines) =>
{
// Only send valid lines
_log.InfoFormat("Forwarding {0} lines.", lines.Length);
udpSender.Send(lines.Where(p => !(p is InvalidMessage)).ToArray());
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = ExecutionDataflowBlockOptions.Unbounded,
CancellationToken = _tokenSource.Token
});
var batchBlock = new BatchBlock <StatsdMessage>(10, new GroupingDataflowBlockOptions()
{
CancellationToken = _tokenSource.Token,
BoundedCapacity = GroupingDataflowBlockOptions.Unbounded
});
batchBlock.LinkTo(outputBlock);
var messageParserBlock = MessageParserBlockFactory.CreateMessageParserBlock(_tokenSource.Token, relayMetrics, _log);
messageParserBlock.LinkTo(batchBlock);
// Completion chain
messageParserBlock.Completion.LogAndContinueWith(_log, "MessageParserBlock",
() =>
{
_log.Info("MessageParserBlock: Completion signalled. Notifying BatchBlock.");
batchBlock.Complete();
});
batchBlock.Completion.LogAndContinueWith(_log, "BatchBlock",
() =>
{
_log.Info("BatchBlock: Completion signalled. Notifying OutputBlock.");
outputBlock.Complete();
});
outputBlock.Completion.LogAndContinueWith(_log, "OutputBlock",
() =>
{
// Last one to leave the room turns out the lights.
_log.Info("OutputBlock: Completion signalled. Shutting down.");
_shutdownComplete.Set();
});
// Listeners
if (config.listeners.udp.enabled)
{
var udpListener = new UdpStatsListener((int)config.listeners.udp.port, relayMetrics);
udpListener.LinkTo(messageParserBlock, _tokenSource.Token);
_listeners.Add(udpListener);
}
if (config.listeners.http.enabled)
{
var httpListener = new HttpStatsListener((int)config.listeners.http.port, relayMetrics);
httpListener.LinkTo(messageParserBlock, _tokenSource.Token);
_listeners.Add(httpListener);
}
if (config.listeners.tcp.enabled)
{
var tcpListener = new TcpStatsListener((int)config.listeners.tcp.port, relayMetrics);
tcpListener.LinkTo(messageParserBlock, _tokenSource.Token);
_listeners.Add(tcpListener);
}
// Set the system metrics target
relayMetrics.SetTarget(batchBlock);
}
public Statsd(dynamic config)
: this((string)config.general.name)
{
_log.Info("statsd.net loading config.");
var systemMetrics = SuperCheapIOC.Resolve <ISystemMetricsService>();
// Load backends
if (config.backends.console.enabled)
{
AddBackend(new ConsoleBackend(), "console");
}
if (config.backends.graphite.enabled)
{
AddBackend(new GraphiteBackend(config.backends.graphite.host, (int)config.backends.graphite.port, systemMetrics), "graphite");
}
if (config.backends.sqlserver.enabled)
{
AddBackend(new SqlServerBackend(
config.backends.sqlserver.connectionString,
config.general.name,
systemMetrics,
batchSize: (int)config.backends.sqlserver.writeBatchSize),
"sqlserver");
}
// Load Aggregators
var intervalServices = new List <IIntervalService>();
var intervalService = new IntervalService(( int )config.calc.flushIntervalSeconds);
intervalServices.Add(intervalService);
AddAggregator(MessageType.Counter,
TimedCounterAggregatorBlockFactory.CreateBlock(_messageBroadcaster, config.calc.countersNamespace, intervalService, _log));
AddAggregator(MessageType.Gauge,
TimedGaugeAggregatorBlockFactory.CreateBlock(_messageBroadcaster, config.calc.gaugesNamespace, config.calc.deleteGaugesOnFlush, intervalService, _log));
AddAggregator(MessageType.Set,
TimedSetAggregatorBlockFactory.CreateBlock(_messageBroadcaster, config.calc.setsNamespace, intervalService, _log));
AddAggregator(MessageType.Timing,
TimedLatencyAggregatorBlockFactory.CreateBlock(_messageBroadcaster, config.calc.timersNamespace, intervalService, _log));
// Load Latency Percentile Aggregators
foreach (var percentile in (IDictionary <string, object>)config.calc.percentiles)
{
dynamic thePercentile = percentile.Value;
intervalService = new IntervalService((int)thePercentile.flushIntervalSeconds);
AddAggregator(MessageType.Timing,
TimedLatencyPercentileAggregatorBlockFactory.CreateBlock(_messageBroadcaster, config.calc.timersNamespace,
intervalService,
(int)thePercentile.percentile,
percentile.Key,
_log));
intervalServices.Add(intervalService);
}
// Load listeners - done last and once the rest of the chain is in place
if (config.listeners.udp.enabled)
{
AddListener(new UdpStatsListener((int)config.listeners.udp.port, systemMetrics));
}
if (config.listeners.tcp.enabled)
{
AddListener(new TcpStatsListener((int)config.listeners.tcp.port, systemMetrics));
}
if (config.listeners.http.enabled)
{
AddListener(new HttpStatsListener((int)config.listeners.http.port, systemMetrics));
}
// Now start the interval service
intervalServices.ForEach(p => p.Start());
}