public static HerculesSinkSettings SetupMetrics([NotNull] this HerculesSinkSettings settings, [NotNull] IMetricContext metricContext)
{
var oldSetup = settings.AdditionalSetup;
settings.AdditionalSetup = setup =>
{
oldSetup?.Invoke(setup);
var newModule = new MetricsModule(metricContext);
var oldModules = setup.Modules.Values.SelectMany(v => v.Before);
var oldModule = oldModules.OfType <MetricsModule>().FirstOrDefault();
if (oldModule != null)
{
oldModule.AddNestedModule(newModule);
}
else
{
setup.AddRequestModule(newModule);
}
};
return(settings);
}
public static IMetricContext Get([NotNull] HerculesSinkSettings settings, [CanBeNull] ILog log)
{
if (defaultMetricContext != null)
{
return(defaultMetricContext);
}
lock (Sync)
{
if (defaultMetricContext != null)
{
return(defaultMetricContext);
}
var memoryLimit = 8 * 1024 * 1024;
var sinkSettings = new HerculesSinkSettings(settings.Cluster, settings.ApiKeyProvider)
{
AdditionalSetup = settings.AdditionalSetup,
MaximumMemoryConsumption = memoryLimit,
MaximumPerStreamMemoryConsumption = memoryLimit
};
log = (log ?? LogProvider.Get()).WithMinimumLevel(LogLevel.Warn);
var sink = new HerculesSink(sinkSettings, log);
var metricsSender = new HerculesMetricSender(new HerculesMetricSenderSettings(sink));
return(defaultMetricContext = new MetricContext(new MetricContextConfig(metricsSender)));
}
}
public StreamStateFactory([NotNull] HerculesSinkSettings settings, [NotNull] ILog log)
{
this.settings = settings ?? throw new ArgumentNullException(nameof(settings));
this.log = log;
globalMemoryManager = new MemoryManager(settings.MaximumMemoryConsumption);
}
public StreamJobFactory([NotNull] HerculesSinkSettings settings, [NotNull] ILog log)
{
this.log = log;
senderFactory = new StreamSenderFactory(settings, log);
plannerFactory = new PlannerFactory(settings);
requestTimeout = settings.RequestTimeout;
}
public static void InitializeHerculesLogger(string command)
{
if (!(herculesLog is null))
{
return;
}
var configLogFilePath = Path.Combine(Helper.GetCementInstallDirectory(), "dotnet", "herculeslog.config.json");
if (!File.Exists(configLogFilePath))
{
ConsoleWriter.WriteError($"{configLogFilePath} not found.");
return;
}
var configuration = new ConfigurationBuilder()
.AddJsonFile(configLogFilePath)
.Build()
.Get <HerculesLogConfiguration>();
if (!configuration.Enabled)
{
return;
}
var settings = new HerculesSinkSettings(new FixedClusterProvider(configuration.ServerUrl), () => configuration.ApiKey)
{
MaximumMemoryConsumption = configuration.MaximumMemoryConsumptionInBytes
};
var fileLogForHercules = GetFileLogger("hercules");
var herculesSink = new HerculesSink(settings, fileLogForHercules);
herculesLog = new HerculesLog(new HerculesLogSettings(herculesSink, configuration.Stream))
.WithProperties(
new Dictionary <string, object>
{
["project"] = configuration.Project,
["environment"] = configuration.Environment,
["hostName"] = ObtainHostname(),
["command"] = command
});
disposables.Add(herculesSink);
loggerFactory.AddVostok(herculesLog);
}
private Hercules()
{
var log = new SynchronousConsoleLog();
Cluster = HerculesCluster.DeployNew(TestContext.CurrentContext.TestDirectory, log.WithMinimumLevel(LogLevel.Warn));
string GetApiKey() => Cluster.ApiKey;
var managementSettings = new HerculesManagementClientSettings(
Cluster.HerculesManagementApiTopology,
GetApiKey);
var streamSettings = new HerculesStreamClientSettings(
Cluster.HerculesStreamApiTopology,
GetApiKey);
var metricsStreamSettings = new HerculesStreamClientSettings <MetricEvent>(
Cluster.HerculesStreamApiTopology,
GetApiKey,
buffer => new HerculesMetricEventReader(buffer));
var gateSettings = new HerculesGateClientSettings(
Cluster.HerculesGateTopology,
GetApiKey);
var sinkSettings = new HerculesSinkSettings(
Cluster.HerculesGateTopology,
GetApiKey)
{
SendPeriod = 1.Seconds()
};
Management = new HerculesManagementClient(
managementSettings,
log);
Sink = new HerculesSink(sinkSettings, log);
Stream = new HerculesStreamClient(streamSettings, log);
MetricsStream = new HerculesStreamClient <MetricEvent>(metricsStreamSettings, log);
Gate = new HerculesGateClient(gateSettings, log);
}
public IHerculesSink Build(BuildContext context)
{
if (!enabled)
{
context.LogDisabled("HerculesSink");
return(null);
}
if (instance != null)
{
return(instance);
}
var cluster = clusterProviderBuilder?.Build(context);
if (cluster == null)
{
context.LogDisabled("HerculesSink", "unconfigured cluster provider");
return(null);
}
var log = context.Log;
if (!verboseLogging)
{
log = log.WithMinimumLevel(LogLevel.Warn);
}
log = log.DropEvents(evt => evt?.MessageTemplate != null && evt.MessageTemplate.Contains("put event to a disposed"));
// Note(kungurtsev): allow null api key provider, streams can be configured later.
var settings = new HerculesSinkSettings(cluster, apiKeyProvider ?? (() => null))
{
AdditionalSetup = setup =>
{
setup.ClientApplicationName = context.ApplicationIdentity.FormatServiceName();
setup.SetupDistributedTracing(context.Tracer);
}
};
settingsCustomization.Customize(settings);
return(new HerculesSink(settings, log));
}
public void Setup()
{
transport = Substitute.For <ITransport>();
transport.Capabilities.Returns(
TransportCapabilities.RequestStreaming |
TransportCapabilities.ResponseStreaming |
TransportCapabilities.RequestCompositeBody);
var settings = new HerculesSinkSettings(new FixedClusterProvider(new Uri("http://localhost/")), () => "apiKey")
{
AdditionalSetup = c => c.Transport = transport,
SuppressVerboseLogging = false,
SendPeriod = 100.Milliseconds()
};
sink = new HerculesSink(settings, new SynchronousConsoleLog());
SetResponse(Responses.Ok);
}
public StreamSenderFactory(HerculesSinkSettings settings, ILog log)
{
this.log = log;
var bufferPool = new BufferPool(settings.MaximumBatchSize * 2, settings.MaxParallelStreams * 8);
apiKeyProvider = settings.ApiKeyProvider;
contentFactory = new RequestContentFactory(bufferPool);
snapshotBatcher = new BufferSnapshotBatcher(settings.MaximumBatchSize);
requestSender = new GateRequestSender(
settings.Cluster,
settings.SuppressVerboseLogging
? log.WithMinimumLevel(LogLevel.Warn).WithLevelsTransformation(SuppressVerboseLoggingLevelsTransformation)
: log,
bufferPool,
settings.AdditionalSetup);
responseAnalyzer = new ResponseAnalyzer(ResponseAnalysisContext.Stream);
statusAnalyzer = new StatusAnalyzer();
}
public Hercules()
{
log = new SynchronousConsoleLog();
cluster = HerculesCluster.DeployNew(TestContext.CurrentContext.TestDirectory, log.WithMinimumLevel(LogLevel.Warn));
string GetApiKey() => cluster.ApiKey;
var managementSettings = new HerculesManagementClientSettings(
cluster.HerculesManagementApiTopology,
GetApiKey);
var streamSettings = new HerculesStreamClientSettings(
cluster.HerculesStreamApiTopology,
GetApiKey);
var gateSettings = new HerculesGateClientSettings(
cluster.HerculesGateTopology,
GetApiKey);
var sinkSettings = new HerculesSinkSettings(
cluster.HerculesGateTopology,
GetApiKey)
{
SendPeriod = 1.Seconds()
};
Management = new HerculesManagementClient(
managementSettings,
log);
Sink = new HerculesSink(sinkSettings, log);
Stream = new HerculesStreamClient(streamSettings, log);
Gate = new HerculesGateClient(gateSettings, log);
}
public PlannerFactory(HerculesSinkSettings settings) => this.settings = settings;
public static void Validate(HerculesSinkSettings settings)
{
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (settings.MaximumMemoryConsumption <= 0)
{
throw new ArgumentException($"Maximum memory consumption has incorrect value {settings.MaximumMemoryConsumption}");
}
if (settings.MaximumPerStreamMemoryConsumption <= 0)
{
throw new ArgumentException($"Maximum per-stream memory consumption has incorrect value {settings.MaximumPerStreamMemoryConsumption}");
}
if (settings.MaximumBatchSize <= 0)
{
throw new ArgumentException($"Maximum batch size has incorrect value {settings.MaximumBatchSize}");
}
if (settings.MaximumRecordSize <= 0)
{
throw new ArgumentException($"Maximum record size has incorrect value {settings.MaximumRecordSize}");
}
if (settings.MaxParallelStreams <= 0)
{
throw new ArgumentException($"Max parallel streams has incorrect value {settings.MaxParallelStreams}");
}
if (settings.SendPeriod <= TimeSpan.Zero)
{
throw new ArgumentException($"Send period has incorrect value {settings.SendPeriod}");
}
if (settings.SendPeriodCap <= TimeSpan.Zero)
{
throw new ArgumentException($"Send period cap has incorrect value {settings.SendPeriodCap}");
}
if (settings.MaximumPerStreamMemoryConsumption > settings.MaximumMemoryConsumption)
{
throw new ArgumentException($"Maximum per-stream memory consumption {settings.MaximumPerStreamMemoryConsumption} is greater than maximum memory consumption {settings.MaximumMemoryConsumption}");
}
if (settings.MaximumRecordSize > settings.MaximumBatchSize)
{
throw new ArgumentException($"Maximum record size {settings.MaximumRecordSize} is greater than maximum batch size {settings.MaximumBatchSize}.");
}
if (settings.SendPeriod > settings.SendPeriodCap)
{
throw new ArgumentException($"Send period {settings.SendPeriod} is greater than send period cap {settings.SendPeriodCap}.");
}
if (settings.Cluster == null)
{
throw new ArgumentNullException(nameof(settings.Cluster));
}
if (settings.ApiKeyProvider == null)
{
throw new ArgumentNullException(nameof(settings.ApiKeyProvider));
}
}
