public EventHubAdapterFactory(
string name,
EventHubOptions ehOptions,
EventHubReceiverOptions receiverOptions,
EventHubStreamCachePressureOptions cacheOptions,
StreamCacheEvictionOptions cacheEvictionOptions,
StreamStatisticOptions statisticOptions,
IEventHubDataAdapter dataAdapter,
IServiceProvider serviceProvider,
SerializationManager serializationManager,
ITelemetryProducer telemetryProducer,
ILoggerFactory loggerFactory)
{
this.Name = name;
this.cacheEvictionOptions = cacheEvictionOptions ?? throw new ArgumentNullException(nameof(cacheEvictionOptions));
this.statisticOptions = statisticOptions ?? throw new ArgumentNullException(nameof(statisticOptions));
this.ehOptions = ehOptions ?? throw new ArgumentNullException(nameof(ehOptions));
this.cacheOptions = cacheOptions ?? throw new ArgumentNullException(nameof(cacheOptions));
this.dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter));
this.receiverOptions = receiverOptions ?? throw new ArgumentNullException(nameof(receiverOptions));
this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
}
/// <summary>
/// Initialize the factory
/// </summary>
/// <param name="providerConfig"></param>
/// <param name="providerName"></param>
/// <param name="log"></param>
/// <param name="svcProvider"></param>
public void Init(IProviderConfiguration providerConfig, string providerName, Logger log, IServiceProvider svcProvider)
{
logger = log;
serviceProvider = svcProvider;
receivers = new ConcurrentDictionary <QueueId, Receiver>();
adapterConfig = new GeneratorAdapterConfig(providerName);
adapterConfig.PopulateFromProviderConfig(providerConfig);
this.serializationManager = svcProvider.GetRequiredService <SerializationManager>();
this.telemetryProducer = svcProvider.GetService <ITelemetryProducer>();
if (CacheMonitorFactory == null)
{
this.CacheMonitorFactory = (dimensions, telemetryProducer) => new DefaultCacheMonitor(dimensions, telemetryProducer);
}
if (this.BlockPoolMonitorFactory == null)
{
this.BlockPoolMonitorFactory = (dimensions, telemetryProducer) => new DefaultBlockPoolMonitor(dimensions, telemetryProducer);
}
if (this.ReceiverMonitorFactory == null)
{
this.ReceiverMonitorFactory = (dimensions, telemetryProducer) => new DefaultQueueAdapterReceiverMonitor(dimensions, telemetryProducer);
}
if (adapterConfig.GeneratorConfigType != null)
{
generatorConfig = (IStreamGeneratorConfig)(serviceProvider?.GetService(adapterConfig.GeneratorConfigType) ?? Activator.CreateInstance(adapterConfig.GeneratorConfigType));
if (generatorConfig == null)
{
throw new ArgumentOutOfRangeException(nameof(providerConfig), "GeneratorConfigType not valid.");
}
generatorConfig.PopulateFromProviderConfig(providerConfig);
}
this.sharedDimensions = new MonitorAggregationDimensions(serviceProvider.GetService <GlobalConfiguration>(), serviceProvider.GetService <NodeConfiguration>());
}
public SiloStatisticsManager(
IOptions <SiloStatisticsOptions> statisticsOptions,
IOptions <LoadSheddingOptions> loadSheddingOptions,
IOptions <StorageOptions> azureStorageOptions,
ILocalSiloDetails siloDetails,
SerializationManager serializationManager,
ITelemetryProducer telemetryProducer,
IHostEnvironmentStatistics hostEnvironmentStatistics,
IAppEnvironmentStatistics appEnvironmentStatistics,
ILoggerFactory loggerFactory,
IOptions <SiloMessagingOptions> messagingOptions)
{
this.siloDetails = siloDetails;
this.storageOptions = azureStorageOptions.Value;
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(messagingOptions.Value.ResponseTimeout);
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logger = loggerFactory.CreateLogger <SiloStatisticsManager>();
this.hostEnvironmentStatistics = hostEnvironmentStatistics;
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
this.MetricsTable = new SiloPerformanceMetrics(this.hostEnvironmentStatistics, appEnvironmentStatistics, loggerFactory, loadSheddingOptions);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
/// <summary>
/// Factory initialization.
/// </summary>
/// <param name="providerConfig"></param>
/// <param name="name"></param>
/// <param name="svcProvider"></param>
public void Init(IProviderConfiguration providerConfig, string name, IServiceProvider svcProvider)
{
logger = svcProvider.GetService <ILogger <MemoryAdapterFactory <TSerializer> > >();
this.loggerFactory = svcProvider.GetRequiredService <ILoggerFactory>();
serviceProvider = svcProvider;
providerName = name;
queueGrains = new ConcurrentDictionary <QueueId, IMemoryStreamQueueGrain>();
adapterConfig = new MemoryAdapterConfig(providerName);
this.telemetryProducer = svcProvider.GetService <ITelemetryProducer>();
if (CacheMonitorFactory == null)
{
this.CacheMonitorFactory = (dimensions, telemetryProducer) => new DefaultCacheMonitor(dimensions, telemetryProducer);
}
if (this.BlockPoolMonitorFactory == null)
{
this.BlockPoolMonitorFactory = (dimensions, telemetryProducer) => new DefaultBlockPoolMonitor(dimensions, telemetryProducer);
}
if (this.ReceiverMonitorFactory == null)
{
this.ReceiverMonitorFactory = (dimensions, telemetryProducer) => new DefaultQueueAdapterReceiverMonitor(dimensions, telemetryProducer);
}
purgePredicate = new TimePurgePredicate(adapterConfig.DataMinTimeInCache, adapterConfig.DataMaxAgeInCache);
grainFactory = (IGrainFactory)serviceProvider.GetService(typeof(IGrainFactory));
adapterConfig.PopulateFromProviderConfig(providerConfig);
streamQueueMapper = new HashRingBasedStreamQueueMapper(adapterConfig.TotalQueueCount, adapterConfig.StreamProviderName);
this.sharedDimensions = new MonitorAggregationDimensions(serviceProvider.GetService <GlobalConfiguration>(), serviceProvider.GetService <NodeConfiguration>());
this.serializer = MemoryMessageBodySerializerFactory <TSerializer> .GetOrCreateSerializer(svcProvider);
}
public EventHubAdapterReceiver(EventHubPartitionSettings settings,
Func<string, IStreamQueueCheckpointer<string>, Logger, ITelemetryProducer, IEventHubQueueCache> cacheFactory,
Func<string, Task<IStreamQueueCheckpointer<string>>> checkpointerFactory,
Logger baseLogger,
IQueueAdapterReceiverMonitor monitor,
Factory<NodeConfiguration> getNodeConfig,
ITelemetryProducer telemetryProducer,
Func<EventHubPartitionSettings, string, Logger, ITelemetryProducer, Task<IEventHubReceiver>> eventHubReceiverFactory = null)
{
if (settings == null) throw new ArgumentNullException(nameof(settings));
if (cacheFactory == null) throw new ArgumentNullException(nameof(cacheFactory));
if (checkpointerFactory == null) throw new ArgumentNullException(nameof(checkpointerFactory));
if (baseLogger == null) throw new ArgumentNullException(nameof(baseLogger));
if (monitor == null) throw new ArgumentNullException(nameof(monitor));
if (telemetryProducer == null) throw new ArgumentNullException(nameof(telemetryProducer));
this.settings = settings;
this.cacheFactory = cacheFactory;
this.checkpointerFactory = checkpointerFactory;
this.baseLogger = baseLogger;
this.logger = baseLogger.GetSubLogger("receiver", "-");
this.monitor = monitor;
this.getNodeConfig = getNodeConfig;
this.telemetryProducer = telemetryProducer;
this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory;
}
public virtual void Init()
{
this.receivers = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
this.telemetryProducer = this.serviceProvider.GetService <ITelemetryProducer>();
InitEventHubClient();
if (this.CacheFactory == null)
{
this.CacheFactory = CreateCacheFactory(this.cacheOptions).CreateCache;
}
if (this.StreamFailureHandlerFactory == null)
{
//TODO: Add a queue specific default failure handler with reasonable error reporting.
this.StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
if (this.QueueMapperFactory == null)
{
this.QueueMapperFactory = partitions => new EventHubQueueMapper(partitions, this.Name);
}
if (this.ReceiverMonitorFactory == null)
{
this.ReceiverMonitorFactory = (dimensions, logger, telemetryProducer) => new DefaultEventHubReceiverMonitor(dimensions, telemetryProducer);
}
this.logger = this.loggerFactory.CreateLogger($"{this.GetType().FullName}.{this.ehOptions.Path}");
}
public EHPurgeLogicTests()
{
//an mock eh settings
this.ehSettings = new EventHubPartitionSettings
{
Hub = new EventHubOptions(),
Partition = "MockPartition",
ReceiverOptions = new EventHubReceiverOptions()
};
//set up cache pressure monitor and purge predicate
this.cachePressureInjectionMonitor = new CachePressureInjectionMonitor();
this.purgePredicate = new PurgeDecisionInjectionPredicate(TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(30));
//set up serialization env
var environment = SerializationTestEnvironment.InitializeWithDefaults();
this.serializationManager = environment.SerializationManager;
//set up buffer pool, small buffer size make it easy for cache to allocate multiple buffers
var oneKB = 1024;
this.bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(oneKB));
this.telemetryProducer = NullTelemetryProducer.Instance;
}
public TransactionManager(
TransactionLog transactionLog,
IOptions <TransactionsOptions> configOption,
ILoggerFactory loggerFactory,
ITelemetryProducer telemetryProducer,
IOptions <StatisticsOptions> statisticsOptions,
TimeSpan?logMaintenanceInterval = null)
{
this.transactionLog = transactionLog;
this.options = configOption.Value;
this.logger = loggerFactory.CreateLogger <TransactionManager>();
this.logMaintenanceInterval = logMaintenanceInterval ?? DefaultLogMaintenanceInterval;
activeTransactionsTracker = new ActiveTransactionsTracker(configOption, this.transactionLog, loggerFactory);
transactionsTable = new ConcurrentDictionary <long, Transaction>(2, 1000000);
dependencyQueue = new ConcurrentQueue <Transaction>();
groupCommitQueue = new ConcurrentQueue <Tuple <CommitRecord, Transaction> >();
checkpointQueue = new ConcurrentQueue <Transaction>();
this.dependencyLock = new InterlockedExchangeLock();
this.commitLock = new InterlockedExchangeLock();
this.checkpointLock = new InterlockedExchangeLock();
this.resources = new Dictionary <ITransactionalResource, long>();
this.transactions = new List <Transaction>();
this.metrics =
new TransactionManagerMetrics(telemetryProducer, configOption.Value.MetricsWritePeriod);
this.checkpointedLSN = 0;
this.IsRunning = false;
}
public virtual void Init()
{
this.receivers = new ConcurrentDictionary <QueueId, KafkaEventBusAdapterReceiver>();
this.telemetryProducer = this.serviceProvider.GetService <ITelemetryProducer>();
if (producer == null)
{
InitKafkaProducer();
}
_adapterCache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions {
CacheSize = cacheOptions.CacheSize
}, Name, loggerFactory);
if (this.StreamFailureHandlerFactory == null)
{
//TODO: Add a queue specific default failure handler with reasonable error reporting.
this.StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
if (this.QueueMapperFactory == null)
{
var h = new HashRingStreamQueueMapperOptions {
TotalQueueCount = receiverOptions.TotalQueueCount
};
this.streamQueueMapper = new HashRingBasedStreamQueueMapper(h, Name);
this.QueueMapperFactory = () => new HashRingBasedStreamQueueMapper(h, this.Name);
}
this.logger = this.loggerFactory.CreateLogger($"{this.GetType().FullName}"); // join topics?
}
public TransactionAgentStatisticsReporter(ITransactionAgentStatistics statistics, ITelemetryProducer telemetryProducer, IOptions <StatisticsOptions> options)
{
this.statistics = statistics ?? throw new ArgumentNullException(nameof(statistics));
this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(statistics));
this.statisticsOptions = options.Value;
this.lastReported = TransactionAgentStatistics.Copy(statistics);
this.lastReportTime = DateTime.UtcNow;
}
public void TrackMetric(ITelemetryProducer telemetryProducer)
{
var rawValue = GetCurrentValue();
var value = valueConverter?.Invoke(rawValue) ?? rawValue;
telemetryProducer.TrackMetric(currentName, value);
// TODO: track delta, when we figure out how to calculate them accurately
}
public DefaultBlockPoolMonitor(BlockPoolMonitorDimensions dimensions, ITelemetryProducer telemetryProducer)
: this(telemetryProducer)
{
this.LogProperties = new Dictionary <string, string>
{
{ "BlockPoolId", dimensions.BlockPoolId },
};
}
public DefaultCacheMonitor(CacheMonitorDimensions dimensions, ITelemetryProducer telemetryProducer)
: this(telemetryProducer)
{
this.LogProperties = new Dictionary <string, string>
{
{ "QueueId", dimensions.QueueId },
};
}
public EventHubAdapterFactory(string name, EventHubStreamOptions options, IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
{
this.Name = name;
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
}
public DefaultCacheMonitor(CacheMonitorDimensions dimensions, ITelemetryProducer telemetryProducer)
: this(telemetryProducer)
{
this.LogProperties = new Dictionary <string, string>
{
{ "QueueId", dimensions.QueueId },
{ "HostName", dimensions.NodeConfig.HostNameOrIPAddress }
};
}
public MemoryAdapterFactory(string providerName, MemoryStreamOptions options, IServiceProvider serviceProvider, IGrainFactory grainFactory, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
{
this.Name = providerName;
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.grainFactory = grainFactory ?? throw new ArgumentNullException(nameof(grainFactory));
this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
this.logger = loggerFactory.CreateLogger <ILogger <MemoryAdapterFactory <TSerializer> > >();
this.serializer = MemoryMessageBodySerializerFactory <TSerializer> .GetOrCreateSerializer(serviceProvider);
}
public GeneratorAdapterFactory(string providerName, GeneratedStreamOptions options, IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
{
this.Name = providerName;
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
this.serializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
this.logger = loggerFactory.CreateLogger <GeneratorAdapterFactory>();
}
/// <summary>
/// Factory initialization.
/// Provider config must contain the event hub settings type or the settings themselves.
/// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
/// </summary>
/// <param name="providerCfg"></param>
/// <param name="providerName"></param>
/// <param name="log"></param>
/// <param name="svcProvider"></param>
public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider)
{
if (providerCfg == null)
{
throw new ArgumentNullException(nameof(providerCfg));
}
if (string.IsNullOrWhiteSpace(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (log == null)
{
throw new ArgumentNullException(nameof(log));
}
providerConfig = providerCfg;
serviceProvider = svcProvider;
this.loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>();
receivers = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
this.SerializationManager = this.serviceProvider.GetRequiredService <SerializationManager>();
adapterSettings = new EventHubStreamProviderSettings(providerName);
adapterSettings.PopulateFromProviderConfig(providerConfig);
hubSettings = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider);
this.telemetryProducer = serviceProvider.GetService <ITelemetryProducer>();
InitEventHubClient();
if (CheckpointerFactory == null)
{
checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider);
CheckpointerFactory = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition, this.loggerFactory);
}
if (CacheFactory == null)
{
CacheFactory = CreateCacheFactory(adapterSettings).CreateCache;
}
if (StreamFailureHandlerFactory == null)
{
//TODO: Add a queue specific default failure handler with reasonable error reporting.
StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
if (QueueMapperFactory == null)
{
QueueMapperFactory = partitions => new EventHubQueueMapper(partitions, adapterSettings.StreamProviderName);
}
if (ReceiverMonitorFactory == null)
{
ReceiverMonitorFactory = (dimensions, logger, telemetryProducer) => new DefaultEventHubReceiverMonitor(dimensions, telemetryProducer);
}
logger = log.GetLogger($"EventHub.{hubSettings.Path}");
}
public GeneratorAdapterFactory(string providerName, HashRingStreamQueueMapperOptions queueMapperOptions, StreamStatisticOptions statisticOptions, IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
{
this.Name = providerName;
this.queueMapperOptions = queueMapperOptions ?? throw new ArgumentNullException(nameof(queueMapperOptions));
this.statisticOptions = statisticOptions ?? throw new ArgumentNullException(nameof(statisticOptions));
this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
this.serializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
this.logger = loggerFactory.CreateLogger <GeneratorAdapterFactory>();
}
public SiloStatisticsManager(
IOptions <StatisticsOptions> statisticsOptions,
ITelemetryProducer telemetryProducer,
ILoggerFactory loggerFactory)
{
MessagingStatisticsGroup.Init();
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init();
StorageStatisticsGroup.Init();
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, loggerFactory);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
public MemoryAdapterFactory(string providerName, StreamCacheEvictionOptions cacheOptions, StreamStatisticOptions statisticOptions, HashRingStreamQueueMapperOptions queueMapperOptions,
IServiceProvider serviceProvider, IGrainFactory grainFactory, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
{
this.Name = providerName;
this.queueMapperOptions = queueMapperOptions ?? throw new ArgumentNullException(nameof(queueMapperOptions));
this.cacheOptions = cacheOptions ?? throw new ArgumentNullException(nameof(cacheOptions));
this.statisticOptions = statisticOptions ?? throw new ArgumentException(nameof(statisticOptions));
this.grainFactory = grainFactory ?? throw new ArgumentNullException(nameof(grainFactory));
this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
this.logger = loggerFactory.CreateLogger <ILogger <MemoryAdapterFactory <TSerializer> > >();
this.serializer = MemoryMessageBodySerializerFactory <TSerializer> .GetOrCreateSerializer(serviceProvider);
}
/// <summary>
/// Initialize the counter publisher framework. Start the background stats writer thread.
/// </summary>
/// <param name="writeInterval">Frequency of writing to Windows perf counters</param>
/// <param name="telemetryProducer">The metrics writer.</param>
public CountersStatistics(TimeSpan writeInterval, ITelemetryProducer telemetryProducer)
{
if (writeInterval <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(nameof(writeInterval), "Creating CounterStatsPublisher with negative or zero writeInterval");
}
if (telemetryProducer == null)
{
throw new ArgumentNullException(nameof(telemetryProducer));
}
PerfCountersWriteInterval = writeInterval;
this.telemetryProducer = telemetryProducer;
}
public KafkaEventBusAdapterFactory(string name, KafkaOptions kafkaOptions, KafkaReceiverOptions receiverOptions, KafkaStreamCachePressureOptions cacheOptions, StreamStatisticOptions statisticOptions,
IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
{
this.Name = name;
this.statisticOptions = statisticOptions ?? throw new ArgumentNullException(nameof(statisticOptions));
this.kafkaOptions = kafkaOptions ?? throw new ArgumentNullException(nameof(kafkaOptions));
this.cacheOptions = cacheOptions ?? throw new ArgumentNullException(nameof(cacheOptions));
this.receiverOptions = receiverOptions ?? throw new ArgumentNullException(nameof(receiverOptions));
this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
this.Direction = kafkaOptions.Direction;
}
public TestEventHubStreamAdapterFactory(
string name,
EventHubOptions ehOptions,
EventHubReceiverOptions receiverOptions,
EventHubStreamCachePressureOptions cacheOptions,
StreamCacheEvictionOptions evictionOptions,
StreamStatisticOptions statisticOptions,
IEventHubDataAdapter dataAdapter,
IServiceProvider serviceProvider,
ITelemetryProducer telemetryProducer,
ILoggerFactory loggerFactory)
: base(name, ehOptions, receiverOptions, cacheOptions, evictionOptions, statisticOptions, dataAdapter, serviceProvider, telemetryProducer, loggerFactory)
{
StreamFailureHandlerFactory = qid => TestAzureTableStorageStreamFailureHandler.Create(this.serviceProvider.GetRequiredService <Serializer <StreamSequenceToken> >());
}
public EventHubAdapterReceiver(EventHubPartitionSettings settings,
Func <string, IStreamQueueCheckpointer <string>, ILoggerFactory, ITelemetryProducer, IEventHubQueueCache> cacheFactory,
Func <string, Task <IStreamQueueCheckpointer <string> > > checkpointerFactory,
ILoggerFactory loggerFactory,
IQueueAdapterReceiverMonitor monitor,
LoadSheddingOptions loadSheddingOptions,
ITelemetryProducer telemetryProducer,
Func <EventHubPartitionSettings, string, ILogger, ITelemetryProducer, IEventHubReceiver> eventHubReceiverFactory = null)
{
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (cacheFactory == null)
{
throw new ArgumentNullException(nameof(cacheFactory));
}
if (checkpointerFactory == null)
{
throw new ArgumentNullException(nameof(checkpointerFactory));
}
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
if (monitor == null)
{
throw new ArgumentNullException(nameof(monitor));
}
if (loadSheddingOptions == null)
{
throw new ArgumentNullException(nameof(loadSheddingOptions));
}
if (telemetryProducer == null)
{
throw new ArgumentNullException(nameof(telemetryProducer));
}
this.settings = settings;
this.cacheFactory = cacheFactory;
this.checkpointerFactory = checkpointerFactory;
this.loggerFactory = loggerFactory;
this.logger = this.loggerFactory.CreateLogger($"{this.GetType().FullName}.{settings.Hub.Path}.{settings.Partition}");
this.monitor = monitor;
this.telemetryProducer = telemetryProducer;
this.loadSheddingOptions = loadSheddingOptions;
this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory;
}
public EventDataGeneratorAdapterFactory(
string name,
EventDataGeneratorStreamOptions options,
EventHubOptions ehOptions,
EventHubReceiverOptions receiverOptions,
EventHubStreamCachePressureOptions cacheOptions,
StreamCacheEvictionOptions evictionOptions,
StreamStatisticOptions statisticOptions,
IEventHubDataAdapter dataAdapter,
IServiceProvider serviceProvider,
ITelemetryProducer telemetryProducer,
ILoggerFactory loggerFactory)
: base(name, ehOptions, receiverOptions, cacheOptions, evictionOptions, statisticOptions, dataAdapter, serviceProvider, telemetryProducer, loggerFactory)
{
this.ehGeneratorOptions = options;
}
public SiloStatisticsManager(
IOptions <SiloStatisticsOptions> statisticsOptions,
SerializationManager serializationManager,
ITelemetryProducer telemetryProducer,
ILoggerFactory loggerFactory)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init();
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
public EHStreamProviderWithCreatedCacheListAdapterFactory(
string name,
EventDataGeneratorStreamOptions options,
EventHubOptions ehOptions,
EventHubReceiverOptions receiverOptions,
EventHubStreamCachePressureOptions cacheOptions,
StreamCacheEvictionOptions evictionOptions,
StreamStatisticOptions statisticOptions,
IEventHubDataAdapter dataAdatper,
IServiceProvider serviceProvider, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
: base(name, options, ehOptions, receiverOptions, cacheOptions, evictionOptions, statisticOptions, dataAdatper, serviceProvider, telemetryProducer, loggerFactory)
{
this.createdCaches = new ConcurrentBag <QueueCacheForTesting>();
this.cacheOptions = cacheOptions;
this.staticticOptions = statisticOptions;
this.ehOptions = ehOptions;
this.evictionOptions = evictionOptions;
}
/// <summary>
/// Default function to be called to create an EventhubQueueCache in IEventHubQueueCacheFactory.CreateCache method. User can
/// override this method to add more customization.
/// </summary>
protected virtual IEventHubQueueCache CreateCache(
string partition,
IEventHubDataAdapter dataAdatper,
StreamStatisticOptions statisticOptions,
StreamCacheEvictionOptions streamCacheEvictionOptions,
IStreamQueueCheckpointer <string> checkpointer,
ILoggerFactory loggerFactory,
IObjectPool <FixedSizeBuffer> bufferPool,
string blockPoolId,
TimePurgePredicate timePurge,
EventHubMonitorAggregationDimensions sharedDimensions,
ITelemetryProducer telemetryProducer)
{
var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
var cacheMonitor = this.CacheMonitorFactory(cacheMonitorDimensions, loggerFactory, telemetryProducer);
var logger = loggerFactory.CreateLogger($"{typeof(EventHubQueueCache).FullName}.{sharedDimensions.EventHubPath}.{partition}");
var evictionStrategy = new ChronologicalEvictionStrategy(logger, timePurge, cacheMonitor, statisticOptions.StatisticMonitorWriteInterval);
return(new EventHubQueueCache(partition, EventHubAdapterReceiver.MaxMessagesPerRead, bufferPool, dataAdatper, evictionStrategy, checkpointer, logger,
cacheMonitor, statisticOptions.StatisticMonitorWriteInterval, streamCacheEvictionOptions.MetadataMinTimeInCache));
}
public TransactionAgent(
ILocalSiloDetails siloDetails,
ITransactionManagerService tmService,
ILoggerFactory loggerFactory,
ITelemetryProducer telemetryProducer,
IOptions <TransactionsOptions> options)
: base(Constants.TransactionAgentSystemTargetId, siloDetails.SiloAddress, loggerFactory)
{
logger = loggerFactory.CreateLogger <TransactionAgent>();
this.tmService = tmService;
ReadOnlyTransactionId = 0;
//abortSequenceNumber = 0;
abortLowerBound = 0;
this.loggerFactory = loggerFactory;
abortedTransactions = new ConcurrentDictionary <long, long>();
transactionStartQueue = new ConcurrentQueue <Tuple <TimeSpan, TaskCompletionSource <long> > >();
transactionCommitQueue = new ConcurrentQueue <TransactionInfo>();
commitCompletions = new ConcurrentDictionary <long, TaskCompletionSource <bool> >();
outstandingCommits = new HashSet <long>();
this.metrics = new TransactionAgentMetrics(telemetryProducer, options.Value.MetricsWritePeriod);
}