public void CollectorsCanBeCreated()
{
var collector = new CollectorConfiguration()
.CreateCollector();
Assert.NotNull(collector);
}
public static void Main(string[] args)
{
CommandLine.Parser.Default.ParseArguments <Config>(args)
.WithParsed(options => {
var collectorConfig = new CollectorConfiguration()
.WriteTo.InfluxDB(options.InfluxUri, options.InfluxDatabase, options.InfluxUsername, options.InfluxPassword);
CollectorLog.RegisterErrorHandler((message, exception) => {
Console.Error.WriteLine($"Error when recording influx stats: \"{message}\"");
Console.Error.WriteLine(exception);
});
using (var influxCollector = collectorConfig.CreateCollector()) {
if (options.Mode.Equals(PriceMode, StringComparison.InvariantCultureIgnoreCase))
{
RecordCurrentPrice(options, influxCollector).GetAwaiter().GetResult();
}
else if (options.Mode.Equals(UsageSimpleMode, StringComparison.InvariantCultureIgnoreCase))
{
RecordSimpleHistoricUsage(options, influxCollector).GetAwaiter().GetResult();
}
else if (options.Mode.Equals(UsageDetailedMode, StringComparison.InvariantCultureIgnoreCase))
{
RecordDetailedHistoricUsage(options, influxCollector).GetAwaiter().GetResult();
}
else
{
Console.Error.WriteLine($"Unrecognized mode \"{options.Mode}\" A valid --mode of either \"{PriceMode}\", \"{UsageSimpleMode}\" or \"{UsageDetailedMode}\" must be specified");
Environment.Exit(1);
}
}
});
}
/// <summary>
/// Setups the aggregated metric listener.
/// </summary>
/// <param name="providerGuid">The provider unique identifier.</param>
/// <param name="etlFileName">The name of the etw file from when read data. If null, realtime session will be used.</param>
/// <param name="metricProducedAction">The when metric available.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <param name="activeCollector">The active collector.</param>
private static void SetupAggregatedMetricListener(
Guid providerGuid,
string etlFileName,
Action <ILocalAggregatedMetric> metricProducedAction,
CancellationToken cancellationToken,
out ActiveCollector activeCollector)
{
var providers = new Dictionary <Guid, ProviderConfiguration>
{
{
providerGuid,
new ProviderConfiguration(providerGuid, EtwTraceLevel.Verbose, 0, 0)
}
};
var etwSessionConfig = new CollectorConfiguration(EtwSessionsPrefix + "aggregated-")
{
SessionType = SessionType.Realtime,
Providers = providers
};
activeCollector = new ActiveCollector(etwSessionConfig.Name);
activeCollector.StartCollector(etwSessionConfig);
RawListener etwListener = null;
try
{
etwListener = string.IsNullOrWhiteSpace(etlFileName)
? CreateRealTimeListener(providerGuid, etwSessionConfig.Name, metricProducedAction, 1, cancellationToken)
: CreateFileListener(providerGuid, etlFileName, metricProducedAction, 1, cancellationToken);
// TODO: Better to check providers periodically and retry several times.
if (!ActiveCollector.TryUpdateProviders(etwSessionConfig))
{
Logger.Log(LoggerLevel.Error, LogId, "Main", "Failed to update ETW providers. Terminating.");
return;
}
try
{
etwListener.Process();
}
finally
{
Logger.Log(
cancellationToken.IsCancellationRequested ? LoggerLevel.Info : LoggerLevel.Error,
LogId,
"SetupEtwDataPipeline",
"ETW Thread terminated unexpectedly, typically indicates that the ETW session was stopped.");
}
}
finally
{
if (etwListener != null)
{
etwListener.Dispose();
}
}
}
public DeviceScanner(ILogger <DeviceScanner> logger, IBleReader bleReader, IDeviceDataReader dataReader, IOptions <CollectorConfiguration> config)
{
_bleReader = bleReader;
_dataReader = dataReader;
_logger = logger;
_config = config.Value;
}
public async Task ExecuteAsync_RepeatingReadError_StopService()
{
//Arrange
var config = new CollectorConfiguration
{
CollectorEnabled = true,
BluetoothAdapterName = "test",
ScanIntervalSeconds = 0
};
_mqttClientMock.Setup(m => m.IsConnected).Returns(true);
_configMock.Setup(c => c.Value).Returns(config);
_deviceScannerMock.Setup(d => d.GetDeviceDataAsync(It.IsAny <IList <SensorDevice> >())).Throws(new Exception());
//Act
var btGwService = _serviceProvider.GetService <CollectorService>();
var cts = new CancellationTokenSource();
#pragma warning disable CS4014
// Run as fire & forget
Task.Run(() => btGwService.StartAsync(cts.Token));
#pragma warning restore
await Task.Delay(500);
cts.Cancel();
_deviceScannerMock.Verify(d => d.GetDeviceDataAsync(It.IsAny <IList <SensorDevice> >()),
Times.Exactly(CollectorService.MaxRepeatedReadFailCount), "The service did not stop after specified amount of failed reads");
}
public PipelinedCollectorBatchConfiguration(CollectorConfiguration configuration)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
_configuration = configuration;
}
public CollectorService(ILogger <CollectorService> logger, IOptions <CollectorConfiguration> config, IDeviceScanner deviceScanner, IManagedMqttClient mqttClient)
{
_logger = logger;
_config = config.Value;
_sensorDevices = config.Value.SensorDevices;
_sensorDeviceTypes = config.Value.SensorDeviceTypes;
_deviceScanner = deviceScanner;
_mqttClient = mqttClient;
}
public async Task ExecuteAsync_PublishSensorDeviceTypeTypeData_CorrectlyPublished()
{
var publishedMessages = new List <MqttApplicationMessage>();
var sensorDeviceType = new SensorDeviceType()
{
Id = "DeviceType1",
Sensors = new List <Sensor>()
{
new Sensor()
{
Id = "Sensor1",
}
}
};
//Arrange
var config = new CollectorConfiguration
{
CollectorEnabled = true,
BluetoothAdapterName = "test",
ScanIntervalSeconds = 1,
SensorDevices = new List <SensorDevice>(),
SensorDeviceTypes = new List <SensorDeviceType>()
{
sensorDeviceType
}
};
_mqttClientMock.Setup(m => m.IsConnected).Returns(true);
_mqttClientMock.Setup(m => m.PublishAsync(It.IsAny <MqttApplicationMessage>(), CancellationToken.None)).Callback((MqttApplicationMessage message, CancellationToken ct) => { publishedMessages.Add(message); });
_configMock.Setup(c => c.Value).Returns(config);
//Act
var btGwService = _serviceProvider.GetService <CollectorService>();
var cts = new CancellationTokenSource();
#pragma warning disable CS4014
// Run as fire & forget
Task.Run(() => btGwService.StartAsync(cts.Token).ConfigureAwait(false));
#pragma warning restore CS4014
await Task.Delay(100);
await btGwService.HandleConnectedAsync(new MqttClientConnectedEventArgs(new MqttClientAuthenticateResult()));
cts.Cancel();
// Assert
var topic = MqttMessageHelper.GetSensorDeviceTypeTopic(sensorDeviceType.Id);
var payload = MqttMessageHelper.SerializePayload(sensorDeviceType);
var message = publishedMessages.FirstOrDefault(m => m.Topic.Equals(topic, StringComparison.OrdinalIgnoreCase));
Assert.IsNotNull(message, "MQTT message for type data of sensor device type was not received");
Assert.AreEqual(topic, message?.Topic, "Sensor device type message topic was not correct");
Assert.AreEqual(payload, message?.ConvertPayloadToString(), "Sensor device type message payload was not correct");
}
public GCCollectionCountCollector()
{
Configuration = new CollectorConfiguration(nameof(GCCollectionCountCollector));
MetricNames = new[] { _name };
_genNames = new string[GC.MaxGeneration + 1];
for (var gen = 0; gen <= GC.MaxGeneration; gen++)
{
_genNames[gen] = gen.ToString();
}
}
public CollectorGate(string tableName, DbFactory dbFactory,
CollectorConfiguration collectorConfiguration, CollectorNetConfiguration netConfiguration,
CommonConfiguration commonConfiguration, TimeoutConfiguration timeoutConfiguration)
: base(collectorConfiguration, netConfiguration, commonConfiguration, timeoutConfiguration)
{
Contract.Requires(!string.IsNullOrEmpty(tableName));
Contract.Requires(_dbFactory != null);
_tableName = tableName;
_dbFactory = dbFactory;
}
/// <summary>
/// Create all the readers, transformers and publishers.
/// </summary>
/// <param name="collectorConfig">The configuration to use.</param>
public void Configure(CollectorConfiguration collectorConfig)
{
_config = collectorConfig;
_properties = collectorConfig.Properties;
_readerIds = new List <string>();
foreach (var readerConfig in collectorConfig.Readers)
{
_readerIds.Add(readerConfig.Id);
}
}
/// <summary>
/// Create a Collector, this involved creating all the layers - Readers, Transformers and Publishers.
/// </summary>
/// <param name="collectorConfig">The configuration that defines all the layers.</param>
/// <param name="mapperConfigs">The mappers to use by transformers.</param>
/// <returns></returns>
public static ICollector CreateCollector(CollectorConfiguration collectorConfig)
{
ComponentRegistration.RegisterComponents(collectorConfig);
ComponentRegistration.Build();
var collector = ComponentRegistration.CreateInstance <ICollector>(collectorConfig.Type);
collector.Configure(collectorConfig);
return(collector);
}
public MetricsCollector GetMetricsCollector(string tag, string type)
{
var collector = new CollectorConfiguration()
.Tag.With(tag, type)
.Batch.AtInterval(TimeSpan.FromSeconds(1))
.WriteTo.InfluxDB(_config.Hostname, _config.Database, _config.User, _config.Password)
.CreateCollector();
return(collector);
}
public GCCollectionCountCollector(string prefixName)
{
_name = prefixName + "dotnet_collection_count_total";
Configuration = new CollectorConfiguration(nameof(GCCollectionCountCollector));
MetricNames = new[] { _name };
_genNames = new string[GC.MaxGeneration + 1];
for (var gen = 0; gen <= GC.MaxGeneration; gen++)
{
_genNames[gen] = gen.ToString();
}
}
private InfluxManager(IInfluxConfig config)
{
var collector = new CollectorConfiguration()
.Tag.With("influx_manager_version", VERSION)
.Tag.With("host", Environment.MachineName)
.Tag.With("user", Environment.UserName)
.Batch.AtInterval(config.BatchInterval)
.WriteTo.InfluxDB(config.Url, database: config.DatabaseName)
.CreateCollector();
InfluxDB.Collector.Metrics.Collector = collector;
}
public async Task GetDeviceDataAsync_BluetoothDevices_ScanMade()
{
var sensorDeviceTypeId = "RuuviTag";
// Arrange
var sensorDevice = new SensorDevice
{
SensorDeviceTypeId = sensorDeviceTypeId,
Id = "SensorDevice1"
};
var sensorDeviceType = new SensorDeviceType()
{
Id = sensorDeviceTypeId,
Sensors = new List <Sensor>()
{
new Sensor()
{
Id = "Sensor1",
}
}
};
_dataReaderMock.Setup(d => d.ReadDeviceDataAsync(It.IsAny <SensorDevice>()))
.ReturnsAsync(new List <MeasurementData>());
var config = new CollectorConfiguration
{
CollectorEnabled = true,
BluetoothAdapterName = "test",
ScanIntervalSeconds = 1,
SensorDevices = new List <SensorDevice>()
{
sensorDevice
},
SensorDeviceTypes = new List <SensorDeviceType>()
{
sensorDeviceType
}
};
_configMock.Setup(c => c.Value).Returns(config);
var deviceScanner = _serviceProvider.GetService <Collector.DeviceScanner.DeviceScanner>();
// Act
var fullList = await deviceScanner.GetDeviceDataAsync(config.SensorDevices);
// Assert
_bleReaderMock.Verify(b => b.ScanAsync(It.IsAny <string>(), It.IsAny <int>()), Times.Once,
"Bluetooth scan should've been made");
}
internal CollectorConfiguration CreateCollectorConfiguration(ConfigurationCollectorData source)
{
CollectorConfiguration result = new CollectorConfiguration();
if (source.Databases.Count > 0)
{
foreach (var d in source.Databases)
{
result.Databases.Add(d.DatabaseID, Convert(d));
}
}
return(result);
}
private static void InitInflux()
{
string version = ConfigurationManager.AppSettings["Metrics.GlobalContextName"];
var collector = new CollectorConfiguration()
.Tag.With("ver", version)
.Tag.With("host", Environment.MachineName)
.Tag.With("user", Environment.UserName)
.Batch.AtInterval(TimeSpan.FromSeconds(1))
.WriteTo.InfluxDB("http://localhost:8086", database: "playground")
.CreateCollector();
InfluxDB.Collector.Metrics.Collector = collector;
}
private MetricsReporterBuilder(
ITelemetryActivation config,
ITelemetryTagContext tagContext,
string measurementName,
IImmutableDictionary <string, string> tags,
CollectorConfiguration influxConfiguration)
{
_activation = config;
_tagContext = tagContext;
_measurementName = measurementName;
_tags = tags;
_influxConfiguration = influxConfiguration;
}
public void CollectorCanBeDisposedWhileTimerIsWaiting()
{
var written = new TaskCompletionSource <object>();
var collector = new CollectorConfiguration()
.Batch.AtInterval(TimeSpan.FromDays(1))
.WriteTo.Emitter(_ => written.SetResult(null))
.CreateCollector();
collector.Increment("m");
written.Task.Wait();
collector.Dispose();
}
public void DoNotCallFactoryIfCollectorExists()
{
var registry = new CollectorRegistry();
var originalCollector = Substitute.For <ICollector>();
originalCollector.MetricNames.Returns(new[] { "metric" });
var fn = Substitute.For <Func <CollectorConfiguration, ICollector> >();
var cfg = new CollectorConfiguration("testName");
registry.Add("testName", originalCollector);
var result = registry.GetOrAdd(cfg, fn);
Assert.Equal(originalCollector, result);
fn.DidNotReceiveWithAnyArgs();
}
public void CollectorConfiguration_Constructed_Correctly()
{
var config = new CollectorConfiguration();
config.Id = "1";
config.Type = ConfigurationTests.TYPE_COLLECTOR;
config.EndPoints.Should().NotBeNull();
config.Publishers.Should().NotBeNull();
config.Readers.Should().NotBeNull();
config.Properties.Should().NotBeNull();
config.Transformers.Should().NotBeNull();
config.Id.Should().Be("1");
config.Type.Should().Be(ConfigurationTests.TYPE_COLLECTOR);
}
public ProcessCollector(Process process, string prefixName)
{
_cpuSecondsTotalName = prefixName + "process_cpu_seconds_total";
_virtualBytesName = prefixName + "process_virtual_bytes";
_workingSetName = prefixName + "process_working_set";
_privateBytesName = prefixName + "process_private_bytes";
_numThreadsName = prefixName + "process_num_threads";
_processIdName = prefixName + "process_processid";
_startTimeSecondsName = prefixName + "process_start_time_seconds";
_process = process;
Configuration = new CollectorConfiguration(nameof(ProcessCollector));
_processStartTime = ((DateTimeOffset)_process.StartTime.ToUniversalTime()).ToUnixTimeSeconds();
MetricNames = new[] { _cpuSecondsTotalName, _virtualBytesName, _workingSetName, _privateBytesName, _numThreadsName, _processIdName, _startTimeSecondsName };
}
static void Main(string[] args)
{
ICollectorDAL dal = new Ajakka.CollectorMock.DAL();
var config = new CollectorConfiguration();
Console.WriteLine("MessageQueueExchangeName: " + config.MessageQueueExchangeName);
Console.WriteLine("MessageQueueHost: " + config.MessageQueueHost);
Console.WriteLine("DALServerRpcQueueName: " + config.DALServerRpcQueueName);
using (var dalServer = new DALServerMock(config, dal)){
dalServer.Start();
Console.WriteLine("DALServer started");
Console.WriteLine(" Press [enter] to exit.");
Console.ReadLine();
}
}
internal ConfigurationCollectorData Convert(CollectorConfiguration source)
{
ConfigurationCollectorData result = new ConfigurationCollectorData();
if (source != null)
{
if (source.Databases != null)
{
foreach (var d in source.Databases.Values)
{
result.Databases.Add(Convert(d));
}
}
}
return(result);
}
/// <inheritdoc />
/// <summary>
/// Constructors
/// </summary>
public ProcessCollector(Process process)
{
_process = process;
Configuration = new CollectorConfiguration(nameof(ProcessCollector));
_processStartTime = _process.StartTime.ToUniversalTime().ToUnixTimeSeconds();
MetricNames = new[]
{
_cpuSecondsTotalName,
_virtualBytesName,
_workingSetName,
_privateBytesName,
_numThreadsName,
_processIdName,
_startTimeSecondsName,
};
}
/// <summary>
/// Initializes a new instance of the <see cref="MetricsReporterBuilder"/> class.
/// </summary>
/// <param name="activationFactory">The activation factory.</param>
/// <param name="simpleConfig">The simple configuration.</param>
/// <param name="tagContext">The tag context.</param>
public MetricsReporterBuilder(
ITelemetryActivationFactory activationFactory,
ISimpleConfig simpleConfig,
ITelemetryTagContext tagContext)
{
// TODO: move it into Influx provider (InfluxMetricFactory)
string url = simpleConfig["influx-url"];
string db = simpleConfig["influx-database"];
string version = simpleConfig["influx-report-version"];
_activation = activationFactory.Create();
_influxConfiguration = new CollectorConfiguration()
.Tag.With("version", version)
.Tag.With("host", Environment.MachineName)
.Tag.With("user", Environment.UserName)
.Batch.AtInterval(TimeSpan.FromSeconds(5))
.WriteTo.InfluxDB(url, database: db);
_tagContext = tagContext;
}
public async Task GetDeviceDataAsync_NoDataAvailable_CustomMeasurementsNotHandled()
{
var customRule = new CustomMeasurementRule();
// Arrange
var sensorDevice = new SensorDevice
{
SensorDeviceTypeId = "someSensorDeviceTypeId",
Id = "someSensorId",
CalculatedMeasurements = new List <CustomMeasurementRule>()
{
customRule
}
};
_dataReaderMock.Setup(d => d.ReadDeviceDataAsync(It.IsAny <SensorDevice>()))
.ReturnsAsync(new List <MeasurementData>());
var config = new CollectorConfiguration
{
CollectorEnabled = true,
BluetoothAdapterName = "test",
ScanIntervalSeconds = 1,
SensorDevices = new List <SensorDevice>()
{
sensorDevice
},
SensorDeviceTypes = new List <SensorDeviceType>()
};
_configMock.Setup(c => c.Value).Returns(config);
var deviceScanner = _serviceProvider.GetService <Collector.DeviceScanner.DeviceScanner>();
// Act
var fullList = await deviceScanner.GetDeviceDataAsync(config.SensorDevices);
// Assert
_loggerMock.Verify(l => l.Log(LogLevel.Error, It.IsAny <EventId>(), It.IsAny <It.IsAnyType>(), It.IsAny <Exception>(),
(Func <It.IsAnyType, Exception, string>)It.IsAny <object>()), Times.Never, "No error should be raised in DeviceScanner if no measurement data available");
}
public override Task PublishAsync(HealthReport report, CancellationToken cancellationToken)
{
Task.Run(() =>
{
var alert = this.HasToPublishAlert(report);
if (alert)
{
using var collector = new CollectorConfiguration()
.Tag.With("name", _healthCheck.Name)
.Batch.AtInterval(TimeSpan.FromSeconds(2))
.WriteTo.InfluxDB(_influxDBTransportSettings.Host, _influxDBTransportSettings.Database)
.CreateCollector();
var entry = report
.Entries
.FirstOrDefault(x => x.Key == this._healthCheck.Name);
var tags = new Dictionary <string, string>()
{
{ "endpoint", _healthCheck.EndpointOrHost ?? _healthCheck.ConnectionString }
};
var fields = new Dictionary <string, object>()
{
{ "status", (int)entry.Value.Status },
{ "error", entry.Value.Exception },
{ "responsetime", entry.Value.Duration.Milliseconds }
};
foreach (var valueTag in entry.Value.Data)
{
fields.Add(valueTag.Key, valueTag.Value);
}
collector.Write("health_check", fields, tags);
}
});
return(Task.CompletedTask);
}
public void SpecializedCollectorsCanBeCreated()
{
var points = new List <PointData>();
var collector = new CollectorConfiguration()
.WriteTo.Emitter(pts => points.AddRange(pts))
.CreateCollector();
var specialized = collector
.Specialize()
.Tag.With("test", "42")
.CreateCollector();
specialized.Increment("m");
var point = points.Single();
Assert.Equal("42", point.Tags.Single().Value);
Assert.NotNull(specialized);
}
public PipelinedCollectorBatchConfiguration(CollectorConfiguration configuration)
{
if (configuration == null) throw new ArgumentNullException(nameof(configuration));
_configuration = configuration;
}
public PipelinedCollectorTagConfiguration(CollectorConfiguration configuration)
{
if (configuration == null) throw new ArgumentNullException("configuration");
_configuration = configuration;
}