private static async Task SetupAsync()
{
var letterTemplate = new Letter("", "TestRabbitServiceQueue", null, new LetterMetadata());
var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel));
_logger = loggerFactory.CreateLogger <ConsumerDataflow <WorkState> >();
_hashingProvider = new Argon2ID_HashingProvider();
var hashKey = await _hashingProvider.GetHashKeyAsync("passwordforencryption", "saltforencryption", 32).ConfigureAwait(false);
_metricsProvider = new NullMetricsProvider();
_encryptionProvider = new AesGcmEncryptionProvider(hashKey);
_compressionProvider = new LZ4PickleProvider();
_serializationProvider = new Utf8JsonProvider();
_rabbitService = new RabbitService(
"Config.json",
_serializationProvider,
_encryptionProvider,
_compressionProvider,
loggerFactory);
await _rabbitService
.Topologer
.CreateQueueAsync("TestRabbitServiceQueue")
.ConfigureAwait(false);
}
public AvailabilityMetrics(IMetricsProvider metricsProvider, string storageFolder, ISystemTime time = null)
{
this.systemTime = time ?? SystemTime.Instance;
this.availabilities = new List <Availability>();
this.edgeAgent = new Lazy <Availability>(() => new Availability(Constants.EdgeAgentModuleName, this.CalculateEdgeAgentDowntime(), this.systemTime));
Preconditions.CheckNotNull(metricsProvider, nameof(metricsProvider));
this.running = metricsProvider.CreateGauge(
"total_time_running_correctly_seconds",
"The amount of time the module was specified in the deployment and was in the running state",
new List <string> {
"module_name"
});
this.expectedRunning = metricsProvider.CreateGauge(
"total_time_expected_running_seconds",
"The amount of time the module was specified in the deployment",
new List <string> {
"module_name"
});
string storageDirectory = Path.Combine(Preconditions.CheckNonWhiteSpace(storageFolder, nameof(storageFolder)), "availability");
try
{
Directory.CreateDirectory(storageDirectory);
this.checkpointFile = Path.Combine(storageDirectory, "avaliability.checkpoint");
this.updateCheckpointFile = new PeriodicTask(this.UpdateCheckpointFile, this.checkpointFrequency, this.checkpointFrequency, this.log, "Checkpoint Availability");
}
catch (Exception ex)
{
this.log.LogError(ex, "Could not create checkpoint directory");
}
}
public WebApi(
ILogger <IWebApi> logger,
IMessageFactory messageFactory,
ImpostorHqConfig config,
IPasswordFile passwordProvider,
ICryptoManager crypto,
IWebApiUserFactory webApiUserFactory,
IMetricsProvider metricsProvider,
ILogManager logManager)
{
_logger = logger;
_messageFactory = messageFactory;
_passwords = passwordProvider.Passwords;
_crypto = crypto;
_userFactory = webApiUserFactory;
_metricsProvider = metricsProvider;
_logManager = logManager;
_users = new ConcurrentDictionary <IWebSocketConnection, WebApiUser>();
_timeouts = new ConcurrentDictionary <IWebSocketConnection, int>();
_server = new WebSocketServer($"ws://{config.Host}:{config.ApiPort}");
_timer1S = new Timer(1000)
{
AutoReset = true
};
_timer1S.Elapsed += _timer1s_Elapsed;
Start();
}
public bool Start()
{
Startup();
m_log.Info("[ASSETINVENTORY]: Starting AssetInventory Server");
try
{
ConfigFile = AssetInventoryConfig.LoadConfig(ConfigFile);
}
catch (Exception)
{
m_log.Error("[ASSETINVENTORY]: Failed to load the config.");
return(false);
}
StorageProvider = LoadAssetInventoryServerPlugin("/OpenSim/AssetInventoryServer/AssetStorageProvider",
"asset_storage_provider", false) as IAssetStorageProvider;
m_backends.Add(StorageProvider);
InventoryProvider = LoadAssetInventoryServerPlugin("/OpenSim/AssetInventoryServer/InventoryStorageProvider",
"inventory_storage_provider", false) as IInventoryStorageProvider;
m_backends.Add(InventoryProvider);
MetricsProvider = LoadAssetInventoryServerPlugin("/OpenSim/AssetInventoryServer/MetricsProvider",
"metrics_provider", false) as IMetricsProvider;
m_backends.Add(MetricsProvider);
try
{
InitHttpServer((uint)ConfigFile.Configs["Config"].GetInt("listen_port"));
}
catch (Exception ex)
{
m_log.Error("[ASSETINVENTORY]: Initializing the HTTP server failed, shutting down: " + ex.Message);
Shutdown();
return(false);
}
LoadDefaultAssets();
AuthenticationProvider = LoadAssetInventoryServerPlugin("/OpenSim/AssetInventoryServer/AuthenticationProvider",
"authentication_provider", false) as IAuthenticationProvider;
m_backends.Add(AuthenticationProvider);
AuthorizationProvider = LoadAssetInventoryServerPlugin("/OpenSim/AssetInventoryServer/AuthorizationProvider",
"authorization_provider", false) as IAuthorizationProvider;
m_backends.Add(AuthorizationProvider);
m_frontends.AddRange(LoadAssetInventoryServerPlugins("/OpenSim/AssetInventoryServer/Frontend", "frontends"));
// Inform the user if we don't have any frontends at this point.
if (m_frontends.Count == 0)
{
m_log.Info("[ASSETINVENTORY]: Starting with no frontends loaded, which isn't extremely useful. Did you set the 'frontends' configuration parameter?");
}
return(true);
}
public MetricsServiceShould()
{
_counter = new Mock <ICounter>();
_counter
.SetupGet(counter => counter.Name)
.Returns(Fixture.Create <string>());
_metrics = new MetricsProvider(new[] { _counter.Object });
}
public MetricsListener(MetricsListenerConfig listenerConfig, IMetricsProvider metricsProvider)
{
this.listenerConfig = Preconditions.CheckNotNull(listenerConfig, nameof(listenerConfig));
string url = GetMetricsListenerUrlPrefix(listenerConfig);
this.httpListener = new HttpListener();
this.httpListener.Prefixes.Add(url);
this.metricsProvider = Preconditions.CheckNotNull(metricsProvider, nameof(metricsProvider));
}
// Note this requires the prometheus-net library to have been initilized using App.UseMetricServer()
public static void InitWithAspNet(IMetricsProvider metricsProvider, ILogger logger)
{
Preconditions.CheckNotNull(metricsProvider, nameof(metricsProvider));
lock (StateLock)
{
logger.LogInformation("Using Asp Net server for metrics");
Instance = metricsProvider;
}
}
public MetricsListener(MetricsListenerConfig listenerConfig, IMetricsProvider metricsProvider)
{
if (!(metricsProvider is MetricsProvider prometheusMetricsProvider))
{
throw new ArgumentException($"IMetricsProvider of type {metricsProvider.GetType()} is incompatible with {this.GetType()}");
}
this.listenerConfig = Preconditions.CheckNotNull(listenerConfig, nameof(listenerConfig));
this.metricServer = new MetricServer(listenerConfig.Host, listenerConfig.Port, listenerConfig.Suffix.Trim('/') + '/', prometheusMetricsProvider.DefaultRegistry);
}
public MetricsController(ILogger <MetricsController> logger, IMetricsProvider provider = null !)
{
this.logger = logger;
this.provider = provider;
if (provider == null)
{
this.logger.LogInformation("No IMetricsProvider installed, no metrics available. Consider using the Package NetCoreAdmin.Metrics");
}
}
public DeploymentMetrics(IMetricsProvider metricsProvider, string storageFolder, ISystemTime time = null)
{
this.systemTime = time ?? SystemTime.Instance;
this.availabilities = new List <Availability>();
this.edgeAgent = new Lazy <Availability>(() => new Availability(Constants.EdgeAgentModuleName, this.CalculateEdgeAgentDowntime(), this.systemTime));
Preconditions.CheckNotNull(metricsProvider, nameof(metricsProvider));
this.running = metricsProvider.CreateGauge(
"total_time_running_correctly_seconds",
"The amount of time the module was specified in the deployment and was in the running state",
new List <string> {
"module_name", MetricsConstants.MsTelemetry
});
this.expectedRunning = metricsProvider.CreateGauge(
"total_time_expected_running_seconds",
"The amount of time the module was specified in the deployment",
new List <string> {
"module_name", MetricsConstants.MsTelemetry
});
this.unsuccessfulSyncs = metricsProvider.CreateCounter(
"unsuccessful_iothub_syncs",
"The amount of times edgeAgent failed to sync with iotHub",
new List <string> {
MetricsConstants.MsTelemetry
});
this.totalSyncs = metricsProvider.CreateCounter(
"iothub_syncs",
"The amount of times edgeAgent attempted to sync with iotHub, both successful and unsuccessful",
new List <string> {
MetricsConstants.MsTelemetry
});
this.deploymentTime = metricsProvider.CreateHistogram(
"deployment_time_seconds",
"The amount of time it took to complete a new deployment",
new List <string> {
MetricsConstants.MsTelemetry
});
string storageDirectory = Path.Combine(Preconditions.CheckNonWhiteSpace(storageFolder, nameof(storageFolder)), "availability");
try
{
Directory.CreateDirectory(storageDirectory);
this.checkpointFile = Path.Combine(storageDirectory, "avaliability.checkpoint");
this.updateCheckpointFile = new PeriodicTask(this.UpdateCheckpointFile, this.checkpointFrequency, this.checkpointFrequency, this.log, "Checkpoint Availability", false);
}
catch (Exception ex)
{
this.log.LogError(ex, "Could not create checkpoint directory");
}
}
public MetricsRegistration(
IMetricsProvider metricsProvider,
IConfigurationProvider configurationProvider,
IMetricRepository metricRepository,
IMetricMetadataFactory metadataFactory)
{
_metricsProvider = metricsProvider;
_configurationProvider = configurationProvider;
_metricRepository = metricRepository;
_metadataFactory = metadataFactory;
}
public static void InitWithServer(IMetricsProvider metricsProvider, IMetricsListener metricsListener, ILogger logger)
{
Preconditions.CheckNotNull(metricsProvider, nameof(metricsProvider));
Preconditions.CheckNotNull(metricsListener, nameof(metricsListener));
lock (StateLock)
{
Instance = metricsProvider;
Listener = metricsListener;
Listener.Start(logger);
}
}
public OrderApiController(
IDeadlineManager deadlineManager, IOrderManager orderManager,
ILogProvider logProvider,
IMetricsProvider prometheusProvider,
IOptions <KafkaSettings> kafkaSettings)
{
_deadlineManager = deadlineManager;
_orderManager = orderManager;
_logProvider = logProvider;
_kafkaSettings = kafkaSettings;
_prometheusProvider = prometheusProvider;
}
public MetadataMetrics(IMetricsProvider metricsProvider, Func <Task <SystemInfo> > getSystemMetadata)
{
this.getSystemMetadata = Preconditions.CheckNotNull(getSystemMetadata, nameof(getSystemMetadata));
Preconditions.CheckNotNull(metricsProvider, nameof(metricsProvider));
this.metaData = metricsProvider.CreateGauge(
"metadata",
"General metadata about the device. The value is always 0, information is encoded in the tags.",
new List <string> {
"edge_agent_version", "experimental_features", "host_information", MetricsConstants.MsTelemetry
});
}
public ConsumerDataflowService(
IConfiguration config,
ILoggerFactory logger,
IRabbitService rabbitService,
ISerializationProvider serializationProvider,
ICompressionProvider compressionProvider,
IEncryptionProvider encryptionProvider,
IMetricsProvider metricsProvider)
{
_config = config;
_logger = logger.CreateLogger <ConsumerDataflowService>();
_rabbitService = rabbitService;
_serializationProvider = serializationProvider;
_compressionProvider = compressionProvider;
_encryptionProvider = encryptionProvider;
_metricsProvider = metricsProvider;
}
public AvailabilityMetrics(IMetricsProvider metricsProvider, ISystemTime time)
{
this.running = metricsProvider.CreateGauge(
"total_time_running_correctly_seconds",
"The amount of time the module was specified in the deployment and was in the running state",
new List <string> {
"module_name"
});
this.expectedRunning = metricsProvider.CreateGauge(
"total_time_expected_running_seconds",
"The amount of time the module was specified in the deployment",
new List <string> {
"module_name"
});
this.time = time;
this.availabilities = new List <Availability>();
this.edgeAgent = new Lazy <Availability>(() => new Availability("edgeAgent", this.CalculateEdgeAgentDowntime(), this.time));
}
public FactoryMetrics(IMetricsProvider metricsProvider)
{
this.commandCounters = Enum.GetValues(typeof(ModuleCommandMetric)).Cast <ModuleCommandMetric>().ToDictionary(c => c, command =>
{
string commandName = Enum.GetName(typeof(ModuleCommandMetric), command).ToLower();
return(metricsProvider.CreateCounter(
$"module_{commandName}",
"Command sent to module",
new List <string> {
"module_name", "module_version", MetricsConstants.MsTelemetry
}));
});
this.commandTiming = metricsProvider.CreateDuration(
$"command_latency",
"Command sent to module",
new List <string> {
"command", MetricsConstants.MsTelemetry
});
}
public SharedClusterWorker(
ILogger <SharedClusterWorker> logger,
IClusterSettings clusterSettings,
IDescriptorProvider descriptorProvider,
ISharedClusterProviderFactory clusterProvider,
ISharedSetupRootActors setupRootActors = default,
ISubscriptionFactory subscriptionFactory = default,
IMainWorker mainWorker = default,
IMetricsProvider metricsProvider = default
)
{
_logger = logger;
_setupRootActors = setupRootActors;
_clusterSettings = clusterSettings;
_mainWorker = mainWorker;
_descriptorProvider = descriptorProvider;
_clusterProvider = clusterProvider;
_subscriptionFactory = subscriptionFactory;
_metricsProvider = metricsProvider;
_cancellationTokenSource = new CancellationTokenSource();
}
private static void Extract(IMetricsProvider metrics, TextWriter writer, Action <ICounter, TextWriter> extractor)
{
var first = true;
foreach (var counter in metrics.Counters)
{
if (first)
{
first = false;
}
else
{
writer.Write(',');
}
writer.Write('"');
writer.Write(counter.Name);
writer.Write("\":");
extractor(counter, writer);
}
}
private static bool StartUpService()
{
try
{
var builder = new ConfigurationBuilder()
.AddJsonFile(Constant.AppSettingsFile, optional: false, reloadOnChange: true);
_configuration = builder.Build();
_kafkaSettings = new KafkaSettings();
_configuration.GetSection("KafkaSettings").Bind(_kafkaSettings);
var mongoDbSettings = new MongoDbSettings();
_configuration.GetSection("MongoDbSettings").Bind(mongoDbSettings);
var serviceProvider = new ServiceCollection()
.AddSingleton <IPubSubProvider, KafkaProvider>()
.AddSingleton <IOrderServiceManager, OrderServiceManager>()
.AddSingleton <ILogProvider, LogProvider>()
.AddSingleton <IMetricsProvider, PrometheusProvider>()
.AddSingleton <IDataProvider, MongoDBProvider>(settings =>
new MongoDBProvider(mongoDbSettings.Host, mongoDbSettings.Database, mongoDbSettings.OrderCollection))
.Configure <KafkaSettings>(_configuration.GetSection("KafkaSettings"))
.AddOptions()
.BuildServiceProvider();
_pubSubProvider = serviceProvider.GetService <IPubSubProvider>();
_orderServiceManager = serviceProvider.GetService <IOrderServiceManager>();
_logProvider = serviceProvider.GetService <ILogProvider>();
_metricsProvider = serviceProvider.GetService <IMetricsProvider>();
var dbProvider = serviceProvider.GetService <IDataProvider>();
return(true);
}
catch (Exception ex)
{
return(false);
}
}
public ConsumerDataflow <TState> SetMetricsProvider(IMetricsProvider provider)
{
_metricsProvider = provider ?? new NullMetricsProvider();
return(this);
}
public static void Flush(this IMetricsProvider metrics, TextWriter writer)
{
Extract(metrics, writer, (counter, textWriter) => counter.Flush(textWriter));
}
public ExceptionCounter(Dictionary <Type, string> recognizedExceptions, HashSet <Type> ignoredExceptions, IMetricsProvider metricsProvider)
{
this.recognizedExceptions = Preconditions.CheckNotNull(recognizedExceptions, nameof(recognizedExceptions));
this.ignoredExceptions = Preconditions.CheckNotNull(ignoredExceptions, nameof(ignoredExceptions));
Preconditions.CheckNotNull(metricsProvider, nameof(metricsProvider));
this.exceptions = Preconditions.CheckNotNull(metricsProvider.CreateCounter(
"exceptions_total",
"The number of exceptions thrown of the given type",
new List <string> {
"exception_name"
}));
AppDomain.CurrentDomain.FirstChanceException += this.OnException;
}
public MetricsCommandFactory(ICommandFactory underlying, IMetricsProvider metricsProvider)
{
this.underlying = Preconditions.CheckNotNull(underlying, nameof(underlying));
this.factoryMetrics = new FactoryMetrics(metricsProvider);
}
public MetricsInterceptor(IMetricsProvider metricsProvider, ILogger logger)
{
_metricsProvider = metricsProvider;
_logger = logger;
}
public MetricsController(IMetricsProvider metricsProvider, ILogProvider logProvider)
{
_metricsProvider = metricsProvider;
}
/// <summary>
/// Sets the <see cref="IMetricsProvider"/> to retrieve performance metrics for serialized nodes,
/// so that performance metrics are included in the output.
/// </summary>
/// <param name="metricsProvider">Performance metrics provider or <c>null</c> to exclude performance metrics from the output.</param>
public void SetMetricsProvider(IMetricsProvider metricsProvider)
{
_metricsProvider = metricsProvider;
}
public Metrics(IMetricsProvider metricsProvider)
{
_metricsProvider = metricsProvider ?? throw new ArgumentNullException(nameof(metricsProvider));
}
public Sensor(IMetricsProvider metricPovider, ITCPSender client, IOutputLog outputLog)
{
_client = client;
_metricPovider = metricPovider;
_outputLog = outputLog;
}
public SystemResourcesMetrics(IMetricsProvider metricsProvider, Func <Task <SystemResources> > getSystemResources, string apiVersion, TimeSpan updateFrequency)
{
Preconditions.CheckNotNull(metricsProvider, nameof(metricsProvider));
this.getSystemResources = Preconditions.CheckNotNull(getSystemResources, nameof(getSystemResources));
this.apiVersion = Preconditions.CheckNotNull(apiVersion, nameof(apiVersion));
Preconditions.CheckArgument(updateFrequency >= MaxUpdateFrequency, $"Performance metrics cannot update faster than {MaxUpdateFrequency.Humanize()}.");
this.updateFrequency = updateFrequency;
this.hostUptime = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"host_uptime_seconds",
"How long the host has been on",
new List <string> {
MetricsConstants.MsTelemetry
}));
this.iotedgedUptime = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"iotedged_uptime_seconds",
"How long iotedged has been running",
new List <string> {
MetricsConstants.MsTelemetry
}));
this.usedSpace = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"available_disk_space_bytes",
"Amount of space left on the disk",
new List <string> {
"disk_name", "disk_filesystem", "disk_filetype", MetricsConstants.MsTelemetry
}));
this.totalSpace = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"total_disk_space_bytes",
"Size of the disk",
new List <string> {
"disk_name", "disk_filesystem", "disk_filetype", MetricsConstants.MsTelemetry
}));
this.usedMemory = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"used_memory_bytes",
"Amount of RAM used by all processes",
new List <string> {
"module", MetricsConstants.MsTelemetry
}));
this.totalMemory = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"total_memory_bytes",
"RAM available",
new List <string> {
"module", MetricsConstants.MsTelemetry
}));
this.cpuPercentage = Preconditions.CheckNotNull(metricsProvider.CreateHistogram(
"used_cpu_percent",
"Percent of cpu used by all processes",
new List <string> {
"module", MetricsConstants.MsTelemetry
}));
this.createdPids = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"created_pids_total",
"The number of processes or threads the container has created",
new List <string> {
"module", MetricsConstants.MsTelemetry
}));
this.networkIn = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"total_network_in_bytes",
"The amount of bytes recieved from the network",
new List <string> {
"module", MetricsConstants.MsTelemetry
}));
this.networkOut = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"total_network_out_bytes",
"The amount of bytes sent to network",
new List <string> {
"module", MetricsConstants.MsTelemetry
}));
this.diskRead = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"total_disk_read_bytes",
"The amount of bytes read from the disk",
new List <string> {
"module", MetricsConstants.MsTelemetry
}));
this.diskWrite = Preconditions.CheckNotNull(metricsProvider.CreateGauge(
"total_disk_write_bytes",
"The amount of bytes written to disk",
new List <string> {
"module", MetricsConstants.MsTelemetry
}));
}
public bool Start()
{
Startup();
m_log.Info("[ASSETINVENTORY]: Starting AssetInventory Server");
try
{
ConfigFile = AssetInventoryConfig.LoadConfig(ConfigFile);
}
catch (Exception)
{
m_log.Error("[ASSETINVENTORY]: Failed to load the config.");
return false;
}
StorageProvider = LoadAssetInventoryServerPlugin("/OpenSim/AssetInventoryServer/AssetStorageProvider",
"asset_storage_provider", false) as IAssetStorageProvider;
m_backends.Add(StorageProvider);
InventoryProvider = LoadAssetInventoryServerPlugin("/OpenSim/AssetInventoryServer/InventoryStorageProvider",
"inventory_storage_provider", false) as IInventoryStorageProvider;
m_backends.Add(InventoryProvider);
MetricsProvider = LoadAssetInventoryServerPlugin("/OpenSim/AssetInventoryServer/MetricsProvider",
"metrics_provider", false) as IMetricsProvider;
m_backends.Add(MetricsProvider);
try
{
InitHttpServer((uint) ConfigFile.Configs["Config"].GetInt("listen_port"));
}
catch (Exception ex)
{
m_log.Error("[ASSETINVENTORY]: Initializing the HTTP server failed, shutting down: " + ex.Message);
Shutdown();
return false;
}
LoadDefaultAssets();
AuthenticationProvider = LoadAssetInventoryServerPlugin("/OpenSim/AssetInventoryServer/AuthenticationProvider",
"authentication_provider", false) as IAuthenticationProvider;
m_backends.Add(AuthenticationProvider);
AuthorizationProvider = LoadAssetInventoryServerPlugin("/OpenSim/AssetInventoryServer/AuthorizationProvider",
"authorization_provider", false) as IAuthorizationProvider;
m_backends.Add(AuthorizationProvider);
m_frontends.AddRange(LoadAssetInventoryServerPlugins("/OpenSim/AssetInventoryServer/Frontend", "frontends"));
// Inform the user if we don't have any frontends at this point.
if (m_frontends.Count == 0)
m_log.Info("[ASSETINVENTORY]: Starting with no frontends loaded, which isn't extremely useful. Did you set the 'frontends' configuration parameter?");
return true;
}