public EventHubAdapterFactory(
     string name,
     EventHubOptions ehOptions,
     EventHubReceiverOptions receiverOptions,
     EventHubStreamCachePressureOptions cacheOptions,
     StreamCacheEvictionOptions cacheEvictionOptions,
     StreamStatisticOptions statisticOptions,
     IEventHubDataAdapter dataAdapter,
     IServiceProvider serviceProvider,
     SerializationManager serializationManager,
     ITelemetryProducer telemetryProducer,
     ILoggerFactory loggerFactory)
 {
     this.Name = name;
     this.cacheEvictionOptions = cacheEvictionOptions ?? throw new ArgumentNullException(nameof(cacheEvictionOptions));
     this.statisticOptions     = statisticOptions ?? throw new ArgumentNullException(nameof(statisticOptions));
     this.ehOptions            = ehOptions ?? throw new ArgumentNullException(nameof(ehOptions));
     this.cacheOptions         = cacheOptions ?? throw new ArgumentNullException(nameof(cacheOptions));
     this.dataAdapter          = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter));
     this.receiverOptions      = receiverOptions ?? throw new ArgumentNullException(nameof(receiverOptions));
     this.serviceProvider      = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
     this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
     this.telemetryProducer    = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
     this.loggerFactory        = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
 }
Esempio n. 2
0
 /// <summary>
 /// Initialize the factory
 /// </summary>
 /// <param name="providerConfig"></param>
 /// <param name="providerName"></param>
 /// <param name="log"></param>
 /// <param name="svcProvider"></param>
 public void Init(IProviderConfiguration providerConfig, string providerName, Logger log, IServiceProvider svcProvider)
 {
     logger          = log;
     serviceProvider = svcProvider;
     receivers       = new ConcurrentDictionary <QueueId, Receiver>();
     adapterConfig   = new GeneratorAdapterConfig(providerName);
     adapterConfig.PopulateFromProviderConfig(providerConfig);
     this.serializationManager = svcProvider.GetRequiredService <SerializationManager>();
     this.telemetryProducer    = svcProvider.GetService <ITelemetryProducer>();
     if (CacheMonitorFactory == null)
     {
         this.CacheMonitorFactory = (dimensions, telemetryProducer) => new DefaultCacheMonitor(dimensions, telemetryProducer);
     }
     if (this.BlockPoolMonitorFactory == null)
     {
         this.BlockPoolMonitorFactory = (dimensions, telemetryProducer) => new DefaultBlockPoolMonitor(dimensions, telemetryProducer);
     }
     if (this.ReceiverMonitorFactory == null)
     {
         this.ReceiverMonitorFactory = (dimensions, telemetryProducer) => new DefaultQueueAdapterReceiverMonitor(dimensions, telemetryProducer);
     }
     if (adapterConfig.GeneratorConfigType != null)
     {
         generatorConfig = (IStreamGeneratorConfig)(serviceProvider?.GetService(adapterConfig.GeneratorConfigType) ?? Activator.CreateInstance(adapterConfig.GeneratorConfigType));
         if (generatorConfig == null)
         {
             throw new ArgumentOutOfRangeException(nameof(providerConfig), "GeneratorConfigType not valid.");
         }
         generatorConfig.PopulateFromProviderConfig(providerConfig);
     }
     this.sharedDimensions = new MonitorAggregationDimensions(serviceProvider.GetService <GlobalConfiguration>(), serviceProvider.GetService <NodeConfiguration>());
 }
Esempio n. 3
0
 public SiloStatisticsManager(
     IOptions <SiloStatisticsOptions> statisticsOptions,
     IOptions <LoadSheddingOptions> loadSheddingOptions,
     IOptions <StorageOptions> azureStorageOptions,
     ILocalSiloDetails siloDetails,
     SerializationManager serializationManager,
     ITelemetryProducer telemetryProducer,
     IHostEnvironmentStatistics hostEnvironmentStatistics,
     IAppEnvironmentStatistics appEnvironmentStatistics,
     ILoggerFactory loggerFactory,
     IOptions <SiloMessagingOptions> messagingOptions)
 {
     this.siloDetails    = siloDetails;
     this.storageOptions = azureStorageOptions.Value;
     MessagingStatisticsGroup.Init(true);
     MessagingProcessingStatisticsGroup.Init();
     NetworkingStatisticsGroup.Init(true);
     ApplicationRequestsStatisticsGroup.Init(messagingOptions.Value.ResponseTimeout);
     SchedulerStatisticsGroup.Init(loggerFactory);
     StorageStatisticsGroup.Init();
     TransactionsStatisticsGroup.Init();
     this.logger = loggerFactory.CreateLogger <SiloStatisticsManager>();
     this.hostEnvironmentStatistics = hostEnvironmentStatistics;
     this.logStatistics             = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
     this.MetricsTable      = new SiloPerformanceMetrics(this.hostEnvironmentStatistics, appEnvironmentStatistics, loggerFactory, loadSheddingOptions);
     this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
 }
Esempio n. 4
0
        /// <summary>
        /// Factory initialization.
        /// </summary>
        /// <param name="providerConfig"></param>
        /// <param name="name"></param>
        /// <param name="svcProvider"></param>
        public void Init(IProviderConfiguration providerConfig, string name, IServiceProvider svcProvider)
        {
            logger                 = svcProvider.GetService <ILogger <MemoryAdapterFactory <TSerializer> > >();
            this.loggerFactory     = svcProvider.GetRequiredService <ILoggerFactory>();
            serviceProvider        = svcProvider;
            providerName           = name;
            queueGrains            = new ConcurrentDictionary <QueueId, IMemoryStreamQueueGrain>();
            adapterConfig          = new MemoryAdapterConfig(providerName);
            this.telemetryProducer = svcProvider.GetService <ITelemetryProducer>();
            if (CacheMonitorFactory == null)
            {
                this.CacheMonitorFactory = (dimensions, telemetryProducer) => new DefaultCacheMonitor(dimensions, telemetryProducer);
            }
            if (this.BlockPoolMonitorFactory == null)
            {
                this.BlockPoolMonitorFactory = (dimensions, telemetryProducer) => new DefaultBlockPoolMonitor(dimensions, telemetryProducer);
            }
            if (this.ReceiverMonitorFactory == null)
            {
                this.ReceiverMonitorFactory = (dimensions, telemetryProducer) => new DefaultQueueAdapterReceiverMonitor(dimensions, telemetryProducer);
            }
            purgePredicate = new TimePurgePredicate(adapterConfig.DataMinTimeInCache, adapterConfig.DataMaxAgeInCache);
            grainFactory   = (IGrainFactory)serviceProvider.GetService(typeof(IGrainFactory));
            adapterConfig.PopulateFromProviderConfig(providerConfig);
            streamQueueMapper = new HashRingBasedStreamQueueMapper(adapterConfig.TotalQueueCount, adapterConfig.StreamProviderName);

            this.sharedDimensions = new MonitorAggregationDimensions(serviceProvider.GetService <GlobalConfiguration>(), serviceProvider.GetService <NodeConfiguration>());
            this.serializer       = MemoryMessageBodySerializerFactory <TSerializer> .GetOrCreateSerializer(svcProvider);
        }
Esempio n. 5
0
        public EventHubAdapterReceiver(EventHubPartitionSettings settings,
            Func<string, IStreamQueueCheckpointer<string>, Logger, ITelemetryProducer, IEventHubQueueCache> cacheFactory,
            Func<string, Task<IStreamQueueCheckpointer<string>>> checkpointerFactory,
            Logger baseLogger,
            IQueueAdapterReceiverMonitor monitor,
            Factory<NodeConfiguration> getNodeConfig,
            ITelemetryProducer telemetryProducer,
            Func<EventHubPartitionSettings, string, Logger, ITelemetryProducer, Task<IEventHubReceiver>> eventHubReceiverFactory = null)
        {
            if (settings == null) throw new ArgumentNullException(nameof(settings));
            if (cacheFactory == null) throw new ArgumentNullException(nameof(cacheFactory));
            if (checkpointerFactory == null) throw new ArgumentNullException(nameof(checkpointerFactory));
            if (baseLogger == null) throw new ArgumentNullException(nameof(baseLogger));
            if (monitor == null) throw new ArgumentNullException(nameof(monitor));
            if (telemetryProducer == null) throw new ArgumentNullException(nameof(telemetryProducer));
            this.settings = settings;
            this.cacheFactory = cacheFactory;
            this.checkpointerFactory = checkpointerFactory;
            this.baseLogger = baseLogger;
            this.logger = baseLogger.GetSubLogger("receiver", "-");
            this.monitor = monitor;
            this.getNodeConfig = getNodeConfig;
            this.telemetryProducer = telemetryProducer;

            this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory;
        }
Esempio n. 6
0
        public virtual void Init()
        {
            this.receivers         = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
            this.telemetryProducer = this.serviceProvider.GetService <ITelemetryProducer>();

            InitEventHubClient();

            if (this.CacheFactory == null)
            {
                this.CacheFactory = CreateCacheFactory(this.cacheOptions).CreateCache;
            }

            if (this.StreamFailureHandlerFactory == null)
            {
                //TODO: Add a queue specific default failure handler with reasonable error reporting.
                this.StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
            }

            if (this.QueueMapperFactory == null)
            {
                this.QueueMapperFactory = partitions => new EventHubQueueMapper(partitions, this.Name);
            }

            if (this.ReceiverMonitorFactory == null)
            {
                this.ReceiverMonitorFactory = (dimensions, logger, telemetryProducer) => new DefaultEventHubReceiverMonitor(dimensions, telemetryProducer);
            }

            this.logger = this.loggerFactory.CreateLogger($"{this.GetType().FullName}.{this.ehOptions.Path}");
        }
Esempio n. 7
0
        public EHPurgeLogicTests()
        {
            //an mock eh settings
            this.ehSettings = new EventHubPartitionSettings
            {
                Hub             = new EventHubOptions(),
                Partition       = "MockPartition",
                ReceiverOptions = new EventHubReceiverOptions()
            };

            //set up cache pressure monitor and purge predicate
            this.cachePressureInjectionMonitor = new CachePressureInjectionMonitor();
            this.purgePredicate = new PurgeDecisionInjectionPredicate(TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(30));

            //set up serialization env
            var environment = SerializationTestEnvironment.InitializeWithDefaults();

            this.serializationManager = environment.SerializationManager;

            //set up buffer pool, small buffer size make it easy for cache to allocate multiple buffers
            var oneKB = 1024;

            this.bufferPool        = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(oneKB));
            this.telemetryProducer = NullTelemetryProducer.Instance;
        }
Esempio n. 8
0
        public TransactionManager(
            TransactionLog transactionLog,
            IOptions <TransactionsOptions> configOption,
            ILoggerFactory loggerFactory,
            ITelemetryProducer telemetryProducer,
            IOptions <StatisticsOptions> statisticsOptions,
            TimeSpan?logMaintenanceInterval = null)
        {
            this.transactionLog         = transactionLog;
            this.options                = configOption.Value;
            this.logger                 = loggerFactory.CreateLogger <TransactionManager>();
            this.logMaintenanceInterval = logMaintenanceInterval ?? DefaultLogMaintenanceInterval;

            activeTransactionsTracker = new ActiveTransactionsTracker(configOption, this.transactionLog, loggerFactory);

            transactionsTable = new ConcurrentDictionary <long, Transaction>(2, 1000000);

            dependencyQueue  = new ConcurrentQueue <Transaction>();
            groupCommitQueue = new ConcurrentQueue <Tuple <CommitRecord, Transaction> >();
            checkpointQueue  = new ConcurrentQueue <Transaction>();

            this.dependencyLock = new InterlockedExchangeLock();
            this.commitLock     = new InterlockedExchangeLock();
            this.checkpointLock = new InterlockedExchangeLock();
            this.resources      = new Dictionary <ITransactionalResource, long>();
            this.transactions   = new List <Transaction>();
            this.metrics        =
                new TransactionManagerMetrics(telemetryProducer, configOption.Value.MetricsWritePeriod);
            this.checkpointedLSN = 0;
            this.IsRunning       = false;
        }
        public virtual void Init()
        {
            this.receivers         = new ConcurrentDictionary <QueueId, KafkaEventBusAdapterReceiver>();
            this.telemetryProducer = this.serviceProvider.GetService <ITelemetryProducer>();

            if (producer == null)
            {
                InitKafkaProducer();
            }

            _adapterCache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions {
                CacheSize = cacheOptions.CacheSize
            }, Name, loggerFactory);

            if (this.StreamFailureHandlerFactory == null)
            {
                //TODO: Add a queue specific default failure handler with reasonable error reporting.
                this.StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
            }

            if (this.QueueMapperFactory == null)
            {
                var h = new HashRingStreamQueueMapperOptions {
                    TotalQueueCount = receiverOptions.TotalQueueCount
                };
                this.streamQueueMapper = new HashRingBasedStreamQueueMapper(h, Name);

                this.QueueMapperFactory = () => new HashRingBasedStreamQueueMapper(h, this.Name);
            }

            this.logger = this.loggerFactory.CreateLogger($"{this.GetType().FullName}"); // join topics?
        }
Esempio n. 10
0
 public TransactionAgentStatisticsReporter(ITransactionAgentStatistics statistics, ITelemetryProducer telemetryProducer, IOptions <StatisticsOptions> options)
 {
     this.statistics        = statistics ?? throw new ArgumentNullException(nameof(statistics));
     this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(statistics));
     this.statisticsOptions = options.Value;
     this.lastReported      = TransactionAgentStatistics.Copy(statistics);
     this.lastReportTime    = DateTime.UtcNow;
 }
Esempio n. 11
0
        public void TrackMetric(ITelemetryProducer telemetryProducer)
        {
            var rawValue = GetCurrentValue();
            var value    = valueConverter?.Invoke(rawValue) ?? rawValue;

            telemetryProducer.TrackMetric(currentName, value);
            // TODO: track delta, when we figure out how to calculate them accurately
        }
 public DefaultBlockPoolMonitor(BlockPoolMonitorDimensions dimensions, ITelemetryProducer telemetryProducer)
     : this(telemetryProducer)
 {
     this.LogProperties = new Dictionary <string, string>
     {
         { "BlockPoolId", dimensions.BlockPoolId },
     };
 }
Esempio n. 13
0
 public DefaultCacheMonitor(CacheMonitorDimensions dimensions, ITelemetryProducer telemetryProducer)
     : this(telemetryProducer)
 {
     this.LogProperties = new Dictionary <string, string>
     {
         { "QueueId", dimensions.QueueId },
     };
 }
Esempio n. 14
0
 public EventHubAdapterFactory(string name, EventHubStreamOptions options, IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
 {
     this.Name                 = name;
     this.options              = options ?? throw new ArgumentNullException(nameof(options));
     this.serviceProvider      = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
     this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
     this.telemetryProducer    = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
     this.loggerFactory        = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
 }
Esempio n. 15
0
 public DefaultCacheMonitor(CacheMonitorDimensions dimensions, ITelemetryProducer telemetryProducer)
     : this(telemetryProducer)
 {
     this.LogProperties = new Dictionary <string, string>
     {
         { "QueueId", dimensions.QueueId },
         { "HostName", dimensions.NodeConfig.HostNameOrIPAddress }
     };
 }
Esempio n. 16
0
 public MemoryAdapterFactory(string providerName, MemoryStreamOptions options, IServiceProvider serviceProvider, IGrainFactory grainFactory, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
 {
     this.Name              = providerName;
     this.options           = options ?? throw new ArgumentNullException(nameof(options));
     this.grainFactory      = grainFactory ?? throw new ArgumentNullException(nameof(grainFactory));
     this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
     this.loggerFactory     = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
     this.logger            = loggerFactory.CreateLogger <ILogger <MemoryAdapterFactory <TSerializer> > >();
     this.serializer        = MemoryMessageBodySerializerFactory <TSerializer> .GetOrCreateSerializer(serviceProvider);
 }
Esempio n. 17
0
 public GeneratorAdapterFactory(string providerName, GeneratedStreamOptions options, IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
 {
     this.Name                 = providerName;
     this.options              = options ?? throw new ArgumentNullException(nameof(options));
     this.serviceProvider      = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
     this.serializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
     this.telemetryProducer    = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
     this.loggerFactory        = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
     this.logger               = loggerFactory.CreateLogger <GeneratorAdapterFactory>();
 }
Esempio n. 18
0
        /// <summary>
        /// Factory initialization.
        /// Provider config must contain the event hub settings type or the settings themselves.
        /// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
        /// </summary>
        /// <param name="providerCfg"></param>
        /// <param name="providerName"></param>
        /// <param name="log"></param>
        /// <param name="svcProvider"></param>
        public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider)
        {
            if (providerCfg == null)
            {
                throw new ArgumentNullException(nameof(providerCfg));
            }
            if (string.IsNullOrWhiteSpace(providerName))
            {
                throw new ArgumentNullException(nameof(providerName));
            }
            if (log == null)
            {
                throw new ArgumentNullException(nameof(log));
            }

            providerConfig            = providerCfg;
            serviceProvider           = svcProvider;
            this.loggerFactory        = serviceProvider.GetRequiredService <ILoggerFactory>();
            receivers                 = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
            this.SerializationManager = this.serviceProvider.GetRequiredService <SerializationManager>();
            adapterSettings           = new EventHubStreamProviderSettings(providerName);
            adapterSettings.PopulateFromProviderConfig(providerConfig);
            hubSettings            = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider);
            this.telemetryProducer = serviceProvider.GetService <ITelemetryProducer>();

            InitEventHubClient();
            if (CheckpointerFactory == null)
            {
                checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider);
                CheckpointerFactory  = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition, this.loggerFactory);
            }

            if (CacheFactory == null)
            {
                CacheFactory = CreateCacheFactory(adapterSettings).CreateCache;
            }

            if (StreamFailureHandlerFactory == null)
            {
                //TODO: Add a queue specific default failure handler with reasonable error reporting.
                StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
            }

            if (QueueMapperFactory == null)
            {
                QueueMapperFactory = partitions => new EventHubQueueMapper(partitions, adapterSettings.StreamProviderName);
            }

            if (ReceiverMonitorFactory == null)
            {
                ReceiverMonitorFactory = (dimensions, logger, telemetryProducer) => new DefaultEventHubReceiverMonitor(dimensions, telemetryProducer);
            }

            logger = log.GetLogger($"EventHub.{hubSettings.Path}");
        }
Esempio n. 19
0
 public GeneratorAdapterFactory(string providerName, HashRingStreamQueueMapperOptions queueMapperOptions, StreamStatisticOptions statisticOptions, IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
 {
     this.Name = providerName;
     this.queueMapperOptions   = queueMapperOptions ?? throw new ArgumentNullException(nameof(queueMapperOptions));
     this.statisticOptions     = statisticOptions ?? throw new ArgumentNullException(nameof(statisticOptions));
     this.serviceProvider      = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
     this.serializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
     this.telemetryProducer    = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
     this.loggerFactory        = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
     this.logger = loggerFactory.CreateLogger <GeneratorAdapterFactory>();
 }
Esempio n. 20
0
 public SiloStatisticsManager(
     IOptions <StatisticsOptions> statisticsOptions,
     ITelemetryProducer telemetryProducer,
     ILoggerFactory loggerFactory)
 {
     MessagingStatisticsGroup.Init();
     MessagingProcessingStatisticsGroup.Init();
     NetworkingStatisticsGroup.Init();
     StorageStatisticsGroup.Init();
     this.logStatistics     = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, loggerFactory);
     this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
 }
Esempio n. 21
0
 public MemoryAdapterFactory(string providerName, StreamCacheEvictionOptions cacheOptions, StreamStatisticOptions statisticOptions, HashRingStreamQueueMapperOptions queueMapperOptions,
                             IServiceProvider serviceProvider, IGrainFactory grainFactory, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
 {
     this.Name = providerName;
     this.queueMapperOptions = queueMapperOptions ?? throw new ArgumentNullException(nameof(queueMapperOptions));
     this.cacheOptions       = cacheOptions ?? throw new ArgumentNullException(nameof(cacheOptions));
     this.statisticOptions   = statisticOptions ?? throw new ArgumentException(nameof(statisticOptions));
     this.grainFactory       = grainFactory ?? throw new ArgumentNullException(nameof(grainFactory));
     this.telemetryProducer  = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
     this.loggerFactory      = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
     this.logger             = loggerFactory.CreateLogger <ILogger <MemoryAdapterFactory <TSerializer> > >();
     this.serializer         = MemoryMessageBodySerializerFactory <TSerializer> .GetOrCreateSerializer(serviceProvider);
 }
Esempio n. 22
0
        /// <summary>
        /// Initialize the counter publisher framework. Start the background stats writer thread.
        /// </summary>
        /// <param name="writeInterval">Frequency of writing to Windows perf counters</param>
        /// <param name="telemetryProducer">The metrics writer.</param>
        public CountersStatistics(TimeSpan writeInterval, ITelemetryProducer telemetryProducer)
        {
            if (writeInterval <= TimeSpan.Zero)
            {
                throw new ArgumentOutOfRangeException(nameof(writeInterval), "Creating CounterStatsPublisher with negative or zero writeInterval");
            }
            if (telemetryProducer == null)
            {
                throw new ArgumentNullException(nameof(telemetryProducer));
            }

            PerfCountersWriteInterval = writeInterval;
            this.telemetryProducer    = telemetryProducer;
        }
        public KafkaEventBusAdapterFactory(string name, KafkaOptions kafkaOptions, KafkaReceiverOptions receiverOptions, KafkaStreamCachePressureOptions cacheOptions, StreamStatisticOptions statisticOptions,
                                           IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
        {
            this.Name                 = name;
            this.statisticOptions     = statisticOptions ?? throw new ArgumentNullException(nameof(statisticOptions));
            this.kafkaOptions         = kafkaOptions ?? throw new ArgumentNullException(nameof(kafkaOptions));
            this.cacheOptions         = cacheOptions ?? throw new ArgumentNullException(nameof(cacheOptions));
            this.receiverOptions      = receiverOptions ?? throw new ArgumentNullException(nameof(receiverOptions));
            this.serviceProvider      = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
            this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
            this.telemetryProducer    = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
            this.loggerFactory        = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));

            this.Direction = kafkaOptions.Direction;
        }
Esempio n. 24
0
 public TestEventHubStreamAdapterFactory(
     string name,
     EventHubOptions ehOptions,
     EventHubReceiverOptions receiverOptions,
     EventHubStreamCachePressureOptions cacheOptions,
     StreamCacheEvictionOptions evictionOptions,
     StreamStatisticOptions statisticOptions,
     IEventHubDataAdapter dataAdapter,
     IServiceProvider serviceProvider,
     ITelemetryProducer telemetryProducer,
     ILoggerFactory loggerFactory)
     : base(name, ehOptions, receiverOptions, cacheOptions, evictionOptions, statisticOptions, dataAdapter, serviceProvider, telemetryProducer, loggerFactory)
 {
     StreamFailureHandlerFactory = qid => TestAzureTableStorageStreamFailureHandler.Create(this.serviceProvider.GetRequiredService <Serializer <StreamSequenceToken> >());
 }
Esempio n. 25
0
        public EventHubAdapterReceiver(EventHubPartitionSettings settings,
                                       Func <string, IStreamQueueCheckpointer <string>, ILoggerFactory, ITelemetryProducer, IEventHubQueueCache> cacheFactory,
                                       Func <string, Task <IStreamQueueCheckpointer <string> > > checkpointerFactory,
                                       ILoggerFactory loggerFactory,
                                       IQueueAdapterReceiverMonitor monitor,
                                       LoadSheddingOptions loadSheddingOptions,
                                       ITelemetryProducer telemetryProducer,
                                       Func <EventHubPartitionSettings, string, ILogger, ITelemetryProducer, IEventHubReceiver> eventHubReceiverFactory = null)
        {
            if (settings == null)
            {
                throw new ArgumentNullException(nameof(settings));
            }
            if (cacheFactory == null)
            {
                throw new ArgumentNullException(nameof(cacheFactory));
            }
            if (checkpointerFactory == null)
            {
                throw new ArgumentNullException(nameof(checkpointerFactory));
            }
            if (loggerFactory == null)
            {
                throw new ArgumentNullException(nameof(loggerFactory));
            }
            if (monitor == null)
            {
                throw new ArgumentNullException(nameof(monitor));
            }
            if (loadSheddingOptions == null)
            {
                throw new ArgumentNullException(nameof(loadSheddingOptions));
            }
            if (telemetryProducer == null)
            {
                throw new ArgumentNullException(nameof(telemetryProducer));
            }
            this.settings            = settings;
            this.cacheFactory        = cacheFactory;
            this.checkpointerFactory = checkpointerFactory;
            this.loggerFactory       = loggerFactory;
            this.logger              = this.loggerFactory.CreateLogger($"{this.GetType().FullName}.{settings.Hub.Path}.{settings.Partition}");
            this.monitor             = monitor;
            this.telemetryProducer   = telemetryProducer;
            this.loadSheddingOptions = loadSheddingOptions;

            this.eventHubReceiverFactory = eventHubReceiverFactory == null ? EventHubAdapterReceiver.CreateReceiver : eventHubReceiverFactory;
        }
 public EventDataGeneratorAdapterFactory(
     string name,
     EventDataGeneratorStreamOptions options,
     EventHubOptions ehOptions,
     EventHubReceiverOptions receiverOptions,
     EventHubStreamCachePressureOptions cacheOptions,
     StreamCacheEvictionOptions evictionOptions,
     StreamStatisticOptions statisticOptions,
     IEventHubDataAdapter dataAdapter,
     IServiceProvider serviceProvider,
     ITelemetryProducer telemetryProducer,
     ILoggerFactory loggerFactory)
     : base(name, ehOptions, receiverOptions, cacheOptions, evictionOptions, statisticOptions, dataAdapter, serviceProvider, telemetryProducer, loggerFactory)
 {
     this.ehGeneratorOptions = options;
 }
 public SiloStatisticsManager(
     IOptions <SiloStatisticsOptions> statisticsOptions,
     SerializationManager serializationManager,
     ITelemetryProducer telemetryProducer,
     ILoggerFactory loggerFactory)
 {
     MessagingStatisticsGroup.Init(true);
     MessagingProcessingStatisticsGroup.Init();
     NetworkingStatisticsGroup.Init(true);
     ApplicationRequestsStatisticsGroup.Init();
     SchedulerStatisticsGroup.Init(loggerFactory);
     StorageStatisticsGroup.Init();
     TransactionsStatisticsGroup.Init();
     this.logStatistics     = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
     this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
 }
        public EHStreamProviderWithCreatedCacheListAdapterFactory(
            string name,
            EventDataGeneratorStreamOptions options,
            EventHubOptions ehOptions,
            EventHubReceiverOptions receiverOptions,
            EventHubStreamCachePressureOptions cacheOptions,
            StreamCacheEvictionOptions evictionOptions,
            StreamStatisticOptions statisticOptions,
            IEventHubDataAdapter dataAdatper,
            IServiceProvider serviceProvider, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
            : base(name, options, ehOptions, receiverOptions, cacheOptions, evictionOptions, statisticOptions, dataAdatper, serviceProvider, telemetryProducer, loggerFactory)

        {
            this.createdCaches    = new ConcurrentBag <QueueCacheForTesting>();
            this.cacheOptions     = cacheOptions;
            this.staticticOptions = statisticOptions;
            this.ehOptions        = ehOptions;
            this.evictionOptions  = evictionOptions;
        }
        /// <summary>
        /// Default function to be called to create an EventhubQueueCache in IEventHubQueueCacheFactory.CreateCache method. User can
        /// override this method to add more customization.
        /// </summary>
        protected virtual IEventHubQueueCache CreateCache(
            string partition,
            IEventHubDataAdapter dataAdatper,
            StreamStatisticOptions statisticOptions,
            StreamCacheEvictionOptions streamCacheEvictionOptions,
            IStreamQueueCheckpointer <string> checkpointer,
            ILoggerFactory loggerFactory,
            IObjectPool <FixedSizeBuffer> bufferPool,
            string blockPoolId,
            TimePurgePredicate timePurge,
            EventHubMonitorAggregationDimensions sharedDimensions,
            ITelemetryProducer telemetryProducer)
        {
            var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
            var cacheMonitor           = this.CacheMonitorFactory(cacheMonitorDimensions, loggerFactory, telemetryProducer);
            var logger           = loggerFactory.CreateLogger($"{typeof(EventHubQueueCache).FullName}.{sharedDimensions.EventHubPath}.{partition}");
            var evictionStrategy = new ChronologicalEvictionStrategy(logger, timePurge, cacheMonitor, statisticOptions.StatisticMonitorWriteInterval);

            return(new EventHubQueueCache(partition, EventHubAdapterReceiver.MaxMessagesPerRead, bufferPool, dataAdatper, evictionStrategy, checkpointer, logger,
                                          cacheMonitor, statisticOptions.StatisticMonitorWriteInterval, streamCacheEvictionOptions.MetadataMinTimeInCache));
        }
Esempio n. 30
0
        public TransactionAgent(
            ILocalSiloDetails siloDetails,
            ITransactionManagerService tmService,
            ILoggerFactory loggerFactory,
            ITelemetryProducer telemetryProducer,
            IOptions <TransactionsOptions> options)
            : base(Constants.TransactionAgentSystemTargetId, siloDetails.SiloAddress, loggerFactory)
        {
            logger                = loggerFactory.CreateLogger <TransactionAgent>();
            this.tmService        = tmService;
            ReadOnlyTransactionId = 0;
            //abortSequenceNumber = 0;
            abortLowerBound    = 0;
            this.loggerFactory = loggerFactory;

            abortedTransactions    = new ConcurrentDictionary <long, long>();
            transactionStartQueue  = new ConcurrentQueue <Tuple <TimeSpan, TaskCompletionSource <long> > >();
            transactionCommitQueue = new ConcurrentQueue <TransactionInfo>();
            commitCompletions      = new ConcurrentDictionary <long, TaskCompletionSource <bool> >();
            outstandingCommits     = new HashSet <long>();
            this.metrics           = new TransactionAgentMetrics(telemetryProducer, options.Value.MetricsWritePeriod);
        }