public ColdStorageProcessor( Func<string, IBlobWriter> blobWriterFactory, IColdStorageInstrumentationPublisher instrumentationPublisher, CancellationToken token, int warningLevel, int tripLevel, TimeSpan stallInterval, TimeSpan logCooldownInterval, string eventHubName, int maxBlocks = MaxBlocks, int maxBlockSize = MaxBlockSize) { Guard.ArgumentNotNull(blobWriterFactory, "blobWriterFactory"); _blobWriterFactory = blobWriterFactory; _token = token; _warningLevel = warningLevel; _tripLevel = tripLevel; _stallInterval = stallInterval; _logCooldownInterval = logCooldownInterval; _eventHubName = eventHubName; _instrumentationPublisher = instrumentationPublisher; _maxBlockSize = maxBlockSize; _buffers = BufferManager.CreateBufferManager(maxBlocks, _maxBlockSize); _eventHubBufferDataList = new List<BufferedFrameData>(); }
public RollingBlobWriter(IBlobNamingStrategy namingStrategy, IColdStorageInstrumentationPublisher instrumentationPublisher, CloudStorageAccount storageAccount, string containerName, int rollSizeMb, int blocksAllowed = MaxBlocksAllowed, int blockSize = MaxBlockSize) { Guard.ArgumentNotNullOrEmpty(containerName, "containerName"); Guard.ArgumentGreaterOrEqualThan(1, blocksAllowed, "blocksAllowed"); Guard.ArgumentLowerOrEqualThan(MaxBlocksAllowed, blocksAllowed, "blocksAllowed"); Guard.ArgumentGreaterOrEqualThan(1, blockSize, "blockSize"); Guard.ArgumentLowerOrEqualThan(MaxBlockSize, blockSize, "blockSize"); Guard.ArgumentGreaterOrEqualThan(1, rollSizeMb, "rollSizeMb"); Guard.ArgumentNotNull(instrumentationPublisher, "instrumentationPublisher"); _rollSizeBytes = rollSizeMb * (long)MegaBytes; Guard.ArgumentLowerOrEqualThan(blocksAllowed * (long)blockSize, _rollSizeBytes, "rollSizeMb"); _blobClient = storageAccount.CreateCloudBlobClient(); _containerName = containerName; _namingStrategy = namingStrategy; _instrumentationPublisher = instrumentationPublisher; _blocksAllowed = blocksAllowed; _blockSize = blockSize; }
public ColdStorageEventProcessorFactory( Func<string, IBlobWriter> blobWriterFactory, IColdStorageInstrumentationPublisher instrumentationPublisher, CancellationToken token, int warningLevel, int tripLevel, TimeSpan stallInterval, TimeSpan logCooldownInterval, string eventHubName) { Guard.ArgumentNotNull(blobWriterFactory, "blobWriterFactory"); Guard.ArgumentNotNull(instrumentationPublisher, "instrumentationPublisher"); _token = token; _blobWriterFactory = blobWriterFactory; _instrumentationPublisher = instrumentationPublisher; _warningLevel = warningLevel; _tripLevel = tripLevel; _stallInterval = stallInterval; _logCooldownInterval = logCooldownInterval; _eventHubName = eventHubName; }
public static async Task<ColdStorageCoordinator> CreateAsync( string hostName, string eventHubName, string consumerGroupName, string eventHubConnectionString, string checkpointStorageAccount, int maxBatchSize, int prefetchCount, TimeSpan receiveTimeout, IReadOnlyList<string> blobWriterStorageAccounts, string containerName, int rollSizeMb, string blobPrefix, int warningLevel, int tripLevel, TimeSpan stallInterval, TimeSpan logCooldownInterval, IColdStorageInstrumentationPublisher instrumentationPublisher) { Logger.Info("Initializing event hub listener for {0} ({1})", eventHubName, consumerGroupName); var storageAccounts = blobWriterStorageAccounts .Select(CloudStorageAccount.Parse) .ToList(); Func<string, IBlobWriter> blobWriterFactory = partitionId => new RollingBlobWriter(new PartitionAndDateNamingStrategy(partitionId, blobPrefix), instrumentationPublisher, storageAccounts[Int32.Parse(partitionId) % storageAccounts.Count], containerName, rollSizeMb); var ns = NamespaceManager.CreateFromConnectionString(eventHubConnectionString); try { await ns.GetConsumerGroupAsync(eventHubName, consumerGroupName); } catch (Exception e) { Logger.Error(e, "Invalid consumer group name {0} in event hub {1}", consumerGroupName, eventHubName); throw; } Logger.Info("Found consumer group {1} for {0}", eventHubName, consumerGroupName); var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, eventHubName); var factory = new ColdStorageEventProcessorFactory( blobWriterFactory, instrumentationPublisher, CancellationToken.None, warningLevel, tripLevel, stallInterval, logCooldownInterval, eventHubId ); var options = new EventProcessorOptions() { MaxBatchSize = maxBatchSize, PrefetchCount = prefetchCount, ReceiveTimeOut = receiveTimeout, InvokeProcessorAfterReceiveTimeout = true }; options.ExceptionReceived += (s, e) => Logger.Error( e.Exception, "Error on message processing, action {0}", e.Action); var host = new EventProcessorHost( hostName, consumerGroupName: consumerGroupName, eventHubPath: eventHubName, eventHubConnectionString: eventHubConnectionString, storageConnectionString: checkpointStorageAccount); await host.RegisterEventProcessorFactoryAsync(factory, options); return new ColdStorageCoordinator(host); }