public ColdStorageProcessor(
Func<string, IBlobWriter> blobWriterFactory,
IColdStorageInstrumentationPublisher instrumentationPublisher,
CancellationToken token,
int warningLevel,
int tripLevel,
TimeSpan stallInterval,
TimeSpan logCooldownInterval,
string eventHubName,
int maxBlocks = MaxBlocks,
int maxBlockSize = MaxBlockSize)
{
Guard.ArgumentNotNull(blobWriterFactory, "blobWriterFactory");
_blobWriterFactory = blobWriterFactory;
_token = token;
_warningLevel = warningLevel;
_tripLevel = tripLevel;
_stallInterval = stallInterval;
_logCooldownInterval = logCooldownInterval;
_eventHubName = eventHubName;
_instrumentationPublisher = instrumentationPublisher;
_maxBlockSize = maxBlockSize;
_buffers = BufferManager.CreateBufferManager(maxBlocks, _maxBlockSize);
_eventHubBufferDataList = new List<BufferedFrameData>();
}
public RollingBlobWriter(IBlobNamingStrategy namingStrategy,
IColdStorageInstrumentationPublisher instrumentationPublisher,
CloudStorageAccount storageAccount,
string containerName,
int rollSizeMb,
int blocksAllowed = MaxBlocksAllowed,
int blockSize = MaxBlockSize)
{
Guard.ArgumentNotNullOrEmpty(containerName, "containerName");
Guard.ArgumentGreaterOrEqualThan(1, blocksAllowed, "blocksAllowed");
Guard.ArgumentLowerOrEqualThan(MaxBlocksAllowed, blocksAllowed, "blocksAllowed");
Guard.ArgumentGreaterOrEqualThan(1, blockSize, "blockSize");
Guard.ArgumentLowerOrEqualThan(MaxBlockSize, blockSize, "blockSize");
Guard.ArgumentGreaterOrEqualThan(1, rollSizeMb, "rollSizeMb");
Guard.ArgumentNotNull(instrumentationPublisher, "instrumentationPublisher");
_rollSizeBytes = rollSizeMb * (long)MegaBytes;
Guard.ArgumentLowerOrEqualThan(blocksAllowed * (long)blockSize, _rollSizeBytes, "rollSizeMb");
_blobClient = storageAccount.CreateCloudBlobClient();
_containerName = containerName;
_namingStrategy = namingStrategy;
_instrumentationPublisher = instrumentationPublisher;
_blocksAllowed = blocksAllowed;
_blockSize = blockSize;
}
public ColdStorageEventProcessorFactory(
Func<string, IBlobWriter> blobWriterFactory,
IColdStorageInstrumentationPublisher instrumentationPublisher,
CancellationToken token,
int warningLevel,
int tripLevel,
TimeSpan stallInterval,
TimeSpan logCooldownInterval,
string eventHubName)
{
Guard.ArgumentNotNull(blobWriterFactory, "blobWriterFactory");
Guard.ArgumentNotNull(instrumentationPublisher, "instrumentationPublisher");
_token = token;
_blobWriterFactory = blobWriterFactory;
_instrumentationPublisher = instrumentationPublisher;
_warningLevel = warningLevel;
_tripLevel = tripLevel;
_stallInterval = stallInterval;
_logCooldownInterval = logCooldownInterval;
_eventHubName = eventHubName;
}
public static async Task<ColdStorageCoordinator> CreateAsync(
string hostName,
string eventHubName,
string consumerGroupName,
string eventHubConnectionString,
string checkpointStorageAccount,
int maxBatchSize,
int prefetchCount,
TimeSpan receiveTimeout,
IReadOnlyList<string> blobWriterStorageAccounts,
string containerName,
int rollSizeMb,
string blobPrefix,
int warningLevel,
int tripLevel,
TimeSpan stallInterval,
TimeSpan logCooldownInterval,
IColdStorageInstrumentationPublisher instrumentationPublisher)
{
Logger.Info("Initializing event hub listener for {0} ({1})", eventHubName, consumerGroupName);
var storageAccounts = blobWriterStorageAccounts
.Select(CloudStorageAccount.Parse)
.ToList();
Func<string, IBlobWriter> blobWriterFactory =
partitionId =>
new RollingBlobWriter(new PartitionAndDateNamingStrategy(partitionId, blobPrefix),
instrumentationPublisher,
storageAccounts[Int32.Parse(partitionId) % storageAccounts.Count],
containerName,
rollSizeMb);
var ns = NamespaceManager.CreateFromConnectionString(eventHubConnectionString);
try
{
await ns.GetConsumerGroupAsync(eventHubName, consumerGroupName);
}
catch (Exception e)
{
Logger.Error(e, "Invalid consumer group name {0} in event hub {1}", consumerGroupName, eventHubName);
throw;
}
Logger.Info("Found consumer group {1} for {0}", eventHubName, consumerGroupName);
var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, eventHubName);
var factory = new ColdStorageEventProcessorFactory(
blobWriterFactory,
instrumentationPublisher,
CancellationToken.None,
warningLevel,
tripLevel,
stallInterval,
logCooldownInterval,
eventHubId
);
var options = new EventProcessorOptions()
{
MaxBatchSize = maxBatchSize,
PrefetchCount = prefetchCount,
ReceiveTimeOut = receiveTimeout,
InvokeProcessorAfterReceiveTimeout = true
};
options.ExceptionReceived +=
(s, e) => Logger.Error(
e.Exception,
"Error on message processing, action {0}",
e.Action);
var host = new EventProcessorHost(
hostName,
consumerGroupName: consumerGroupName,
eventHubPath: eventHubName,
eventHubConnectionString: eventHubConnectionString,
storageConnectionString: checkpointStorageAccount);
await host.RegisterEventProcessorFactoryAsync(factory, options);
return new ColdStorageCoordinator(host);
}
