public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics) { _log = SuperCheapIOC.Resolve <ILog>(); _systemMetrics = systemMetrics; var config = new SqlServerConfiguration(configElement.Attribute("connectionString").Value, configElement.ToInt("writeBatchSize")); _connectionString = config.ConnectionString; _collectorName = collectorName; _retries = config.Retries; InitialiseRetryHandling(); _batchBlock = new BatchBlock <GraphiteLine>(config.WriteBatchSize); _actionBlock = new ActionBlock <GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 }); _batchBlock.LinkTo(_actionBlock); _batchBlock.Completion.ContinueWith(p => _actionBlock.Complete()); _actionBlock.Completion.ContinueWith(p => { _isActive = false; }); _completionTask = new Task(() => { _log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete."); _batchBlock.Complete(); _actionBlock.Completion.Wait(); }); }
public SqlServerBackend(string connectionString, string collectorName, ISystemMetricsService systemMetrics, int retries = 3, int batchSize = 50) { _log = SuperCheapIOC.Resolve <ILog>(); _connectionString = connectionString; _collectorName = collectorName; _systemMetrics = systemMetrics; _retries = retries; InitialiseRetryHandling(); _batchBlock = new BatchBlock <GraphiteLine>(batchSize); _actionBlock = new ActionBlock <GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 }); _batchBlock.LinkTo(_actionBlock); _batchBlock.Completion.ContinueWith(p => _actionBlock.Complete()); _actionBlock.Completion.ContinueWith(p => { _isActive = false; }); _completionTask = new Task(() => { _log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete."); _batchBlock.Complete(); _actionBlock.Completion.Wait(); }); }
public Statsd(StatsdnetConfiguration config) : this(config.Name) { _log.Info("statsd.net loading config."); var systemMetrics = SuperCheapIOC.Resolve <ISystemMetricsService>(); systemMetrics.HideSystemStats = config.HideSystemStats; LoadBackends(config, systemMetrics); // Load Aggregators var intervalServices = new List <IIntervalService>(); var intervalService = new IntervalService(config.FlushInterval, _tokenSource.Token); intervalServices.Add(intervalService); LoadAggregators(config, intervalService, _messageBroadcaster, systemMetrics); // Load Listeners LoadListeners(config, _tokenSource.Token, systemMetrics); // Now start the interval service intervalServices.ForEach(p => p.Start()); // Announce that we've started systemMetrics.LogCount("started"); }
public GraphiteBackend(string host, int port, ISystemMetricsService systemMetrics) { _log = SuperCheapIOC.Resolve <ILog>(); var ipAddress = Utility.HostToIPv4Address(host); _client = new UdpClient(); _client.Connect(ipAddress, port); _systemMetrics = systemMetrics; _completionTask = new Task(() => { _isActive = false; }); }
public StatsdnetForwardingClient(string host, int port, ISystemMetricsService systemMetrics, int numRetries = 1, bool enableCompression = true) { _host = host; _port = port; _systemMetrics = systemMetrics; _numRetries = numRetries; _enableCompression = enableCompression; _client = new TcpClient(); _log = SuperCheapIOC.Resolve <ILog>(); }
public StatsdnetTcpListener(int port, ISystemMetricsService systemMetrics) { _systemMetrics = systemMetrics; _log = SuperCheapIOC.Resolve <ILog>(); IsListening = false; _activeConnections = 0; _tcpListener = new TcpListener(IPAddress.Any, port); _decoderBlock = new ActionBlock <DecoderBlockPacket>((data) => { DecodePacketAndForward(data); }, Utility.UnboundedExecution()); }
public void Setup() { SuperCheapIOC.Reset(); _statsd = new Statsd(); _listener = new InAppListener(); _backend = new InAppBackend(); _intervalService = new ControllableIntervalService(); _outputBlock = new OutputBufferBlock <GraphiteLine>(); _client = new StatsdClient.Statsd("", 0, outputChannel: new InAppListenerOutputChannel(_listener)); _statsd.AddListener(_listener); _statsd.AddBackend(_backend); }
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics) { _log = SuperCheapIOC.Resolve <ILog>(); _systemMetrics = systemMetrics; _completionTask = new Task(() => { _isActive = false; }); _batchBlock = new BatchBlock <GraphiteLine>(BATCH_SIZE); _senderBlock = new ActionBlock <GraphiteLine[]>((batch) => SendBatch(batch), Utility.OneAtATimeExecution()); _batchBlock.LinkTo(_senderBlock); _isActive = true; _host = configElement.Attribute("host").Value; _port = configElement.ToInt("port"); }
public Statsd(string serviceName = null) { LoggingBootstrap.Configure(); _log.Info("statsd.net starting."); _tokenSource = new CancellationTokenSource(); _shutdownComplete = new ManualResetEvent(false); SuperCheapIOC.Add(_log); var systemInfoService = new SystemInfoService(); SuperCheapIOC.Add(systemInfoService as ISystemInfoService); serviceName = serviceName ?? systemInfoService.HostName; var systemMetricsService = new SystemMetricsService("statsdnet", serviceName); SuperCheapIOC.Add(systemMetricsService as ISystemMetricsService); /** * The flow is: * Listeners -> * Message Parser -> * router -> * Aggregator -> * Broadcaster -> * Backends */ // Initialise the core blocks _router = new StatsdMessageRouterBlock(); _messageParser = MessageParserBlockFactory.CreateMessageParserBlock(_tokenSource.Token, SuperCheapIOC.Resolve <ISystemMetricsService>(), _log); _messageParser.LinkTo(_router); _messageParser.Completion.LogAndContinueWith(_log, "MessageParser", () => { _log.Info("MessageParser: Completion signaled. Notifying the MessageBroadcaster."); _messageBroadcaster.Complete(); }); _messageBroadcaster = new BroadcastBlock <GraphiteLine>(GraphiteLine.Clone); _messageBroadcaster.Completion.LogAndContinueWith(_log, "MessageBroadcaster", () => { _log.Info("MessageBroadcaster: Completion signaled. Notifying all backends."); _backends.ForEach(q => q.Complete()); }); // Add the broadcaster to the IOC container SuperCheapIOC.Add <BroadcastBlock <GraphiteLine> >(_messageBroadcaster); systemMetricsService.SetTarget(_messageBroadcaster); _backends = new List <IBackend>(); _listeners = new List <IListener>(); }
public void AddBackend(IBackend backend, string name = "") { _log.InfoFormat("Adding backend {0} named '{1}'", backend.GetType().Name, name); _backends.Add(backend); SuperCheapIOC.Add(backend, name); _messageBroadcaster.LinkTo(backend); backend.Completion.LogAndContinueWith(_log, name, () => { if (_backends.All(q => !q.IsActive)) { _shutdownComplete.Set(); } }); }
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics) { _log = SuperCheapIOC.Resolve <ILog>(); _systemMetrics = systemMetrics; _completionTask = new Task(() => { _isActive = false; }); _senderBlock = new ActionBlock <GraphiteLine>((message) => SendLine(message), Utility.UnboundedExecution()); _isActive = true; var config = new GraphiteConfiguration(configElement.Attribute("host").Value, configElement.ToInt("port")); var ipAddress = Utility.HostToIPv4Address(config.Host); _client = new UdpClient(); _client.Connect(ipAddress, config.Port); }
public SignalFxReporter(string baseURI, string apiToken, TimeSpan timeout, IWebRequestorFactory requestor = null) { _log = SuperCheapIOC.Resolve <ILog>(); if (requestor == null) { requestor = new WebRequestorFactory() .WithUri(baseURI + "/v2/datapoint") .WithMethod("POST") .WithContentType("application/x-protobuf") .WithHeader("X-SF-TOKEN", apiToken) .WithTimeout(timeout); } this._requestor = requestor; this._apiToken = apiToken; }
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics) { _completionTask = new Task(() => IsActive = false); _log = SuperCheapIOC.Resolve <ILog>(); _config = parseConfig(configElement); _reporter = new SignalFxReporter(_config.BaseURI, _config.ApiToken, _config.PostTimeout); _processBlock = new ActionBlock <Bucket>(bucket => ProcessBucket(bucket), Utility.UnboundedExecution()); _batchBlock = new BatchBlock <TypeDatapoint>(_config.MaxBatchSize); _outputBlock = new ActionBlock <TypeDatapoint[]>(datapoint => ProcessDatapoints(datapoint), Utility.OneAtATimeExecution()); _batchBlock.LinkTo(_outputBlock); _triggerBatchTimer = new Timer((state) => trigger(state), null, TimeSpan.FromSeconds(1), _config.MaxTimeBetweenBatches); IsActive = true; }
private void SendMetrics(object sender, IntervalFiredEventArgs args) { if (_target == null) { return; } // Get a count of metrics waiting to be sent out var outputBufferCount = SuperCheapIOC.ResolveAll <IBackend>().Sum(p => p.OutputCount); LogGauge("outputBuffer", outputBufferCount); LogGauge("up", 1); var bucket = new CounterBucket(_metrics.ToArray(), args.Epoch, _prefix); _metrics.Clear(); if (!HideSystemStats) { _target.Post(bucket); } }
private void SendMetrics(object sender, IntervalFiredEventArgs args) { if (_target == null) { return; } // Get a count of metrics waiting to be sent out var outputBufferCount = SuperCheapIOC.ResolveAll <IBackend>().Sum(p => p.OutputCount); _target.Post(new GraphiteLine(_prefix + "outputBuffer", outputBufferCount)); _target.Post(new GraphiteLine(_prefix + "up", 1)); var pairs = _metrics.ToArray(); _metrics.Clear(); foreach (var pair in pairs) { _target.Post(new GraphiteLine(_prefix + pair.Key, pair.Value)); } }
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics) { _completionTask = new Task(() => IsActive = false); _log = SuperCheapIOC.Resolve <ILog>(); _systemMetrics = systemMetrics; var config = new LibratoBackendConfiguration( email: configElement.Attribute("email").Value, token: configElement.Attribute("token").Value, numRetries: configElement.ToInt("numRetries"), retryDelay: Utility.ConvertToTimespan(configElement.Attribute("retryDelay").Value), postTimeout: Utility.ConvertToTimespan(configElement.Attribute("postTimeout").Value), maxBatchSize: configElement.ToInt("maxBatchSize"), countersAsGauges: configElement.ToBoolean("countersAsGauges") ); _config = config; _source = collectorName; _serviceVersion = Assembly.GetEntryAssembly().GetName().Version.ToString(); _preprocessorBlock = new ActionBlock <Bucket>(bucket => ProcessBucket(bucket), Utility.UnboundedExecution()); _batchBlock = new BatchBlock <LibratoMetric>(_config.MaxBatchSize); _outputBlock = new ActionBlock <LibratoMetric[]>(lines => PostToLibrato(lines), Utility.OneAtATimeExecution()); _batchBlock.LinkTo(_outputBlock); _client = new RestClient(LIBRATO_API_URL); _client.Authenticator = new HttpBasicAuthenticator(_config.Email, _config.Token); _client.Timeout = (int)_config.PostTimeout.TotalMilliseconds; _retryPolicy = new RetryPolicy <LibratoErrorDetectionStrategy>(_config.NumRetries); _retryPolicy.Retrying += (sender, args) => { _log.Warn(String.Format("Retry {0} failed. Trying again. Delay {1}, Error: {2}", args.CurrentRetryCount, args.Delay, args.LastException.Message), args.LastException); _systemMetrics.LogCount("backends.librato.retry"); }; _retryStrategy = new Incremental(_config.NumRetries, _config.RetryDelay, TimeSpan.FromSeconds(2)); IsActive = true; }
public Relay(dynamic config) { LoggingBootstrap.Configure(); _log.Info("statsdrelay is starting up."); _tokenSource = new CancellationTokenSource(); _shutdownComplete = new ManualResetEvent(false); _listeners = new List <IListener>(); var systemInfoService = new SystemInfoService(); var relayMetrics = new RelayMetricsService("relay", _tokenSource.Token, systemInfoService.HostName); SuperCheapIOC.Add(relayMetrics as ISystemMetricsService); /* Pipeline is * UDPStatsListener * HTTPStatsListener * TCPStatsListener * -> MessageParserBlock * -> BatchBlock * -> UDPRawStatsSender */ var udpSender = new UDPRawStatsSender(config.target.host, (int)config.target.port, relayMetrics); var outputBlock = new ActionBlock <StatsdMessage[]>((lines) => { // Only send valid lines _log.InfoFormat("Forwarding {0} lines.", lines.Length); udpSender.Send(lines.Where(p => !(p is InvalidMessage)).ToArray()); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = ExecutionDataflowBlockOptions.Unbounded, CancellationToken = _tokenSource.Token }); var batchBlock = new BatchBlock <StatsdMessage>(10, new GroupingDataflowBlockOptions() { CancellationToken = _tokenSource.Token, BoundedCapacity = GroupingDataflowBlockOptions.Unbounded }); batchBlock.LinkTo(outputBlock); var messageParserBlock = MessageParserBlockFactory.CreateMessageParserBlock(_tokenSource.Token, relayMetrics, _log); messageParserBlock.LinkTo(batchBlock); // Completion chain messageParserBlock.Completion.LogAndContinueWith(_log, "MessageParserBlock", () => { _log.Info("MessageParserBlock: Completion signalled. Notifying BatchBlock."); batchBlock.Complete(); }); batchBlock.Completion.LogAndContinueWith(_log, "BatchBlock", () => { _log.Info("BatchBlock: Completion signalled. Notifying OutputBlock."); outputBlock.Complete(); }); outputBlock.Completion.LogAndContinueWith(_log, "OutputBlock", () => { // Last one to leave the room turns out the lights. _log.Info("OutputBlock: Completion signalled. Shutting down."); _shutdownComplete.Set(); }); // Listeners if (config.listeners.udp.enabled) { var udpListener = new UdpStatsListener((int)config.listeners.udp.port, relayMetrics); udpListener.LinkTo(messageParserBlock, _tokenSource.Token); _listeners.Add(udpListener); } if (config.listeners.http.enabled) { var httpListener = new HttpStatsListener((int)config.listeners.http.port, relayMetrics); httpListener.LinkTo(messageParserBlock, _tokenSource.Token); _listeners.Add(httpListener); } if (config.listeners.tcp.enabled) { var tcpListener = new TcpStatsListener((int)config.listeners.tcp.port, relayMetrics); tcpListener.LinkTo(messageParserBlock, _tokenSource.Token); _listeners.Add(tcpListener); } // Set the system metrics target relayMetrics.SetTarget(batchBlock); }
public Statsd(dynamic config) : this((string)config.general.name) { _log.Info("statsd.net loading config."); var systemMetrics = SuperCheapIOC.Resolve <ISystemMetricsService>(); // Load backends if (config.backends.console.enabled) { AddBackend(new ConsoleBackend(), "console"); } if (config.backends.graphite.enabled) { AddBackend(new GraphiteBackend(config.backends.graphite.host, (int)config.backends.graphite.port, systemMetrics), "graphite"); } if (config.backends.sqlserver.enabled) { AddBackend(new SqlServerBackend( config.backends.sqlserver.connectionString, config.general.name, systemMetrics, batchSize: (int)config.backends.sqlserver.writeBatchSize), "sqlserver"); } // Load Aggregators var intervalServices = new List <IIntervalService>(); var intervalService = new IntervalService(( int )config.calc.flushIntervalSeconds); intervalServices.Add(intervalService); AddAggregator(MessageType.Counter, TimedCounterAggregatorBlockFactory.CreateBlock(_messageBroadcaster, config.calc.countersNamespace, intervalService, _log)); AddAggregator(MessageType.Gauge, TimedGaugeAggregatorBlockFactory.CreateBlock(_messageBroadcaster, config.calc.gaugesNamespace, config.calc.deleteGaugesOnFlush, intervalService, _log)); AddAggregator(MessageType.Set, TimedSetAggregatorBlockFactory.CreateBlock(_messageBroadcaster, config.calc.setsNamespace, intervalService, _log)); AddAggregator(MessageType.Timing, TimedLatencyAggregatorBlockFactory.CreateBlock(_messageBroadcaster, config.calc.timersNamespace, intervalService, _log)); // Load Latency Percentile Aggregators foreach (var percentile in (IDictionary <string, object>)config.calc.percentiles) { dynamic thePercentile = percentile.Value; intervalService = new IntervalService((int)thePercentile.flushIntervalSeconds); AddAggregator(MessageType.Timing, TimedLatencyPercentileAggregatorBlockFactory.CreateBlock(_messageBroadcaster, config.calc.timersNamespace, intervalService, (int)thePercentile.percentile, percentile.Key, _log)); intervalServices.Add(intervalService); } // Load listeners - done last and once the rest of the chain is in place if (config.listeners.udp.enabled) { AddListener(new UdpStatsListener((int)config.listeners.udp.port, systemMetrics)); } if (config.listeners.tcp.enabled) { AddListener(new TcpStatsListener((int)config.listeners.tcp.port, systemMetrics)); } if (config.listeners.http.enabled) { AddListener(new HttpStatsListener((int)config.listeners.http.port, systemMetrics)); } // Now start the interval service intervalServices.ForEach(p => p.Start()); }