public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (String.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
// Creating an options object with all the config values
_options = new KafkaStreamProviderOptions(config);
if (!_options.UsingExternalMetrics)
{
Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/");
}
if (!_options.IncludeMetrics)
{
Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics();
}
_providerName = providerName;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName);
_logger = logger;
_adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger);
_adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger);
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString))
{
throw new ArgumentException(string.Format("{0} property not set", DataConnectionStringPropertyName));
}
if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId))
{
throw new ArgumentException(string.Format("{0} property not set", DeploymentIdPropertyName));
}
cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue);
numQueues = config.GetIntProperty(NumQueuesPropertyName, NumQueuesDefaultValue);
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
if (StreamFailureHandlerFactory == null)
{
StreamFailureHandlerFactory =
qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
}
}
public AzureQueueAdapter(
TDataAdapter dataAdapter,
SerializationManager serializationManager,
HashRingBasedStreamQueueMapper streamQueueMapper,
string dataConnectionString,
string deploymentId,
string providerName,
TimeSpan?messageVisibilityTimeout = null)
{
if (string.IsNullOrEmpty(dataConnectionString))
{
throw new ArgumentNullException(nameof(dataConnectionString));
}
if (string.IsNullOrEmpty(deploymentId))
{
throw new ArgumentNullException(nameof(deploymentId));
}
this.serializationManager = serializationManager;
DataConnectionString = dataConnectionString;
DeploymentId = deploymentId;
Name = providerName;
MessageVisibilityTimeout = messageVisibilityTimeout;
this.streamQueueMapper = streamQueueMapper;
this.dataAdapter = dataAdapter;
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
if (!config.Properties.TryGetValue(DATA_CONNECTION_STRING, out dataConnectionString))
{
throw new ArgumentException(String.Format("{0} property not set", DATA_CONNECTION_STRING));
}
if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out deploymentId))
{
throw new ArgumentException(String.Format("{0} property not set", DEPLOYMENT_ID));
}
cacheSize = SimpleQueueAdapterCache.ParseSize(config, DEFAULT_CACHE_SIZE);
string numQueuesString;
numQueues = DEFAULT_NUM_QUEUES;
if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
{
throw new ArgumentException(String.Format("{0} invalid. Must be int", NUM_QUEUES_PARAM));
}
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
}
/// <summary>
/// Generate default azure queue names
/// </summary>
/// <param name="serviceId"></param>
/// <param name="providerName"></param>
/// <returns></returns>
public static List <string> GenerateDefaultAzureQueueNames(string serviceId, string providerName)
{
var defaultQueueMapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions(), providerName);
return(defaultQueueMapper.GetAllQueues()
.Select(queueName => $"{serviceId}-{queueName}").ToList());
}
private Dictionary <SiloAddress, List <int> > GetQueueHistogram(Dictionary <SiloAddress, List <IRingRangeInternal> > siloRanges, int totalNumQueues)
{
var options = new HashRingStreamQueueMapperOptions();
options.TotalQueueCount = totalNumQueues;
HashRingBasedStreamQueueMapper queueMapper = new HashRingBasedStreamQueueMapper(options, "AzureQueues");
_ = queueMapper.GetAllQueues();
Dictionary <SiloAddress, List <int> > queueHistogram = new Dictionary <SiloAddress, List <int> >();
foreach (var siloRange in siloRanges)
{
List <int> agentRanges = new List <int>();
foreach (IRingRangeInternal agentRange in siloRange.Value)
{
int numQueues = queueMapper.GetQueuesForRange(agentRange).Count();
agentRanges.Add(numQueues);
}
agentRanges.Sort();
queueHistogram.Add(siloRange.Key, agentRanges);
}
//queueHistogram.Sort((t1, t2) => t1.Item2.CompareTo(t2.Item2));
return(queueHistogram);
}
/// <summary> Init the factory.</summary>
public virtual void Init()
{
this.streamQueueMapper = new HashRingBasedStreamQueueMapper(this.options.NumQueues, providerName);
this.adapterCache = new SimpleQueueAdapterCache(this.options.CacheSize, this.providerName, this.loggerFactory);
this.StreamFailureHandlerFactory = this.StreamFailureHandlerFactory ??
((qid) => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler()));
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString))
{
throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName));
}
if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId))
{
throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName));
}
cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue);
string numQueuesString;
numQueues = NumQueuesDefaultValue;
if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
{
throw new ArgumentException(String.Format("{0} invalid. Must be int", NumQueuesPropertyName));
}
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
}
public PubSubAdapter(
TDataAdapter dataAdapter,
ILoggerFactory loggerFactory,
HashRingBasedStreamQueueMapper streamQueueMapper,
string projectId,
string topicId,
string serviceId,
string providerName,
TimeSpan?deadline = null,
string customEndpoint = "")
{
if (string.IsNullOrEmpty(projectId))
{
throw new ArgumentNullException(nameof(projectId));
}
if (string.IsNullOrEmpty(topicId))
{
throw new ArgumentNullException(nameof(topicId));
}
if (string.IsNullOrEmpty(serviceId))
{
throw new ArgumentNullException(nameof(serviceId));
}
_logger = loggerFactory.CreateLogger($"{this.GetType().FullName}.{providerName}");
this.loggerFactory = loggerFactory;
ProjectId = projectId;
TopicId = topicId;
ServiceId = serviceId;
Name = providerName;
Deadline = deadline;
_streamQueueMapper = streamQueueMapper;
_dataAdapter = dataAdapter;
_customEndpoint = customEndpoint;
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (!config.Properties.TryGetValue(AzureQueueAdapterConstants.DataConnectionStringPropertyName, out dataConnectionString))
{
throw new ArgumentException($"{AzureQueueAdapterConstants.DataConnectionStringPropertyName} property not set");
}
if (!config.Properties.TryGetValue(AzureQueueAdapterConstants.DeploymentIdPropertyName, out deploymentId))
{
throw new ArgumentException($"{AzureQueueAdapterConstants.DeploymentIdPropertyName} property not set");
}
string messageVisibilityTimeoutRaw;
if (config.Properties.TryGetValue(AzureQueueAdapterConstants.MessageVisibilityTimeoutPropertyName, out messageVisibilityTimeoutRaw))
{
TimeSpan messageVisibilityTimeoutTemp;
if (!TimeSpan.TryParse(messageVisibilityTimeoutRaw, out messageVisibilityTimeoutTemp))
{
throw new ArgumentException(
$"Failed to parse {AzureQueueAdapterConstants.MessageVisibilityTimeoutPropertyName} value '{messageVisibilityTimeoutRaw}' as a TimeSpan");
}
messageVisibilityTimeout = messageVisibilityTimeoutTemp;
}
else
{
messageVisibilityTimeout = null;
}
cacheSize = SimpleQueueAdapterCache.ParseSize(config, AzureQueueAdapterConstants.CacheSizeDefaultValue);
this.loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>();
string numQueuesString;
numQueues = AzureQueueAdapterConstants.NumQueuesDefaultValue;
if (config.Properties.TryGetValue(AzureQueueAdapterConstants.NumQueuesPropertyName, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
{
throw new ArgumentException($"{AzureQueueAdapterConstants.NumQueuesPropertyName} invalid. Must be int");
}
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
if (StreamFailureHandlerFactory == null)
{
StreamFailureHandlerFactory =
qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
this.SerializationManager = serviceProvider.GetRequiredService <SerializationManager>();
this.adaptorFactory = () => ActivatorUtilities.GetServiceOrCreateInstance <TDataAdapter>(serviceProvider);
}
public virtual void Init()
{
this._streamQueueMapper = new HashRingBasedStreamQueueMapper(this.options.NumSubscriptions, this._providerName);
this._adapterCache = new SimpleQueueAdapterCache(this.options.CacheSize, this._providerName, loggerFactory);
if (StreamFailureHandlerFactory == null)
{
StreamFailureHandlerFactory =
qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString))
{
throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName));
}
if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId))
{
throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName));
}
string messageVisibilityTimeoutRaw;
if (config.Properties.TryGetValue(MessageVisibilityTimeoutPropertyName, out messageVisibilityTimeoutRaw))
{
TimeSpan messageVisibilityTimeoutTemp;
if (!TimeSpan.TryParse(messageVisibilityTimeoutRaw, out messageVisibilityTimeoutTemp))
{
throw new ArgumentException(String.Format("Failed to parse {0} value '{1}' as a TimeSpan",
MessageVisibilityTimeoutPropertyName, messageVisibilityTimeoutRaw));
}
messageVisibilityTimeout = messageVisibilityTimeoutTemp;
}
else
{
messageVisibilityTimeout = null;
}
cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue);
string numQueuesString;
numQueues = NumQueuesDefaultValue;
if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
{
throw new ArgumentException(String.Format("{0} invalid. Must be int", NumQueuesPropertyName));
}
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
if (StreamFailureHandlerFactory == null)
{
StreamFailureHandlerFactory =
qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
}
}
public KafkaQueueAdapterUnitTests()
{
Mock <Logger> loggerMock = new Mock <Logger>();
var connectionStrings = new List <Uri> {
new Uri("http://kafka1:9092;http://kafka2:9092")
};
var topicName = "Jonathan.ab.KafkaStreamProviderTestsNew";
var consumerGroupName = "TestConsumerGroupName";
_logger = loggerMock.Object;
_options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName);
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, _options.TopicName);
}
public void Init(IProviderConfiguration config, string providerName, IServiceProvider serviceProvider)
{
Name = providerName;
_publishSubscribe = serviceProvider.GetRequiredService <IPublishSubscribe>();
_serializationManager = serviceProvider.GetRequiredService <SerializationManager>();
ILoggerFactory loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>();
PublishSubscribeStreamConfiguration publishSubscribeStreamConfiguration = new PublishSubscribeStreamConfiguration(config);
_adapterCache = new SimpleQueueAdapterCache(publishSubscribeStreamConfiguration.InMemoryCacheSize, providerName, loggerFactory);
_streamQueueMapper = new HashRingBasedStreamQueueMapper(publishSubscribeStreamConfiguration.NumberOfQueues, providerName);
}
public RabbitMQAdapter(TDataAdapter dataAdapter,
SerializationManager serializationManager,
HashRingBasedStreamQueueMapper streamQueueMapper,
RabbitMQStreamProviderConfiguration configuration,
string providerName,
Logger logger)
{
_dataAdapter = dataAdapter;
_serializationManager = serializationManager;
_streamQueueMapper = streamQueueMapper;
_configuration = configuration;
_logger = logger;
Name = providerName;
}
public RabbitMessageQueueAdapter(HashRingBasedStreamQueueMapper streamQueueMapper, string dataConnectionString, string deploymentId, string providerName)
{
if (string.IsNullOrEmpty(dataConnectionString))
{
throw new ArgumentNullException("dataConnectionString");
}
if (string.IsNullOrEmpty(deploymentId))
{
throw new ArgumentNullException("deploymentId");
}
DataConnectionString = dataConnectionString;
DeploymentId = deploymentId;
Name = providerName;
_streamQueueMapper = streamQueueMapper;
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
_logger = logger;
_providerName = providerName;
_config = config;
_serviceProvider = serviceProvider;
// Cache size
//string cacheSizeString;
//_cacheSize = DefaultCacheSize;
//if (config.Properties.TryGetValue(CacheSizeParam, out cacheSizeString))
//{
// if (!int.TryParse(cacheSizeString, out _cacheSize))
// throw new ArgumentException($"{CacheSizeParam} invalid. Must be int");
//}
// # queues
string numQueuesString;
_numQueues = DefaultNumQueues;
if (config.Properties.TryGetValue(NumQueuesParam, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out _numQueues))
{
throw new ArgumentException($"{NumQueuesParam} invalid. Must be int");
}
}
// Use Redis for queue?
string useRedis;
_useRedisForQueue = DefaultUseRedisForQueue;
if (config.Properties.TryGetValue(UseRedisForQueueParam, out useRedis))
{
if (!bool.TryParse(useRedis, out _useRedisForQueue))
{
throw new ArgumentException($"{UseRedisForQueueParam} invalid value {useRedis}");
}
}
if (_useRedisForQueue)
{
ReadRedisConnectionParams(config);
}
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_numQueues, providerName);
}
/// <summary>
/// Helper method for testing. Clears all messages in all the queues used by the specifed stream provider.
/// </summary>
/// <param name="providerName">The Azure Queue stream privider name.</param>
/// <param name="deploymentId">The deployment ID hosting the stream provider.</param>
/// <param name="storageConnectionString">The azure storage connection string.</param>
public static async Task ClearAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString)
{
if (deploymentId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterConstants.NumQueuesDefaultValue, providerName);
List <QueueId> allQueues = queueMapper.GetAllQueues().ToList();
var deleteTasks = new List <Task>();
foreach (var queueId in allQueues)
{
var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString);
deleteTasks.Add(manager.ClearQueue());
}
await Task.WhenAll(deleteTasks);
}
}
/// <summary>
/// Helper method for testing. Deletes all the queues used by the specifed stream provider.
/// </summary>
/// <param name="providerName">The Azure Queue stream privider name.</param>
/// <param name="deploymentId">The deployment ID hosting the stream provider.</param>
/// <param name="storageConnectionString">The azure storage connection string.</param>
public static async Task DeleteAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString)
{
if (deploymentId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterFactory.DEFAULT_NUM_QUEUES, providerName);
List <QueueId> allQueues = queueMapper.GetAllQueues().ToList();
var deleteTasks = new List <Task>();
foreach (var queueId in allQueues)
{
var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString);
deleteTasks.Add(manager.DeleteQueue());
}
await Task.WhenAll(deleteTasks);
}
}
public KafkaQueueAdapter(HashRingBasedStreamQueueMapper queueMapper, KafkaStreamProviderOptions options,
string providerName, IKafkaBatchFactory batchFactory, ILogger logger)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (batchFactory == null)
{
throw new ArgumentNullException(nameof(batchFactory));
}
if (queueMapper == null)
{
throw new ArgumentNullException(nameof(queueMapper));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
_options = options;
_streamQueueMapper = queueMapper;
Name = providerName;
_batchFactory = batchFactory;
_logger = logger;
// Creating a producer
KafkaOptions kafkaOptions = new KafkaOptions(_options.ConnectionStrings.ToArray())
{
Log = new KafkaLogBridge(logger)
};
var broker = new BrokerRouter(kafkaOptions);
_producer = new Producer(broker)
{
BatchDelayTime = TimeSpan.FromMilliseconds(_options.TimeToWaitForBatchInMs),
BatchSize = _options.ProduceBatchSize
};
_gateway = new ProtocolGateway(kafkaOptions);
_logger.Info("KafkaQueueAdapter - Created a KafkaQueueAdapter");
}
public SQSAdapterFactory(
string name,
SqsOptions sqsOptions,
HashRingStreamQueueMapperOptions queueMapperOptions,
SimpleQueueCacheOptions cacheOptions,
IOptions <ClusterOptions> clusterOptions,
Orleans.Serialization.Serializer serializer,
ILoggerFactory loggerFactory)
{
this.providerName = name;
this.sqsOptions = sqsOptions;
this.clusterOptions = clusterOptions.Value;
this.serializer = serializer.GetSerializer <SQSBatchContainer>();
this.loggerFactory = loggerFactory;
streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, this.providerName);
adapterCache = new SimpleQueueAdapterCache(cacheOptions, this.providerName, this.loggerFactory);
}
public PubSubAdapterFactory(
string name,
PubSubOptions options,
HashRingStreamQueueMapperOptions queueMapperOptions,
SimpleQueueCacheOptions cacheOptions,
IServiceProvider serviceProvider,
IOptions <ClusterOptions> clusterOptions,
ILoggerFactory loggerFactory)
{
this._providerName = name;
this.options = options;
this.clusterOptions = clusterOptions.Value;
this.loggerFactory = loggerFactory;
this._adaptorFactory = () => ActivatorUtilities.GetServiceOrCreateInstance <TDataAdapter>(serviceProvider);
this._streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, this._providerName);
this._adapterCache = new SimpleQueueAdapterCache(cacheOptions, this._providerName, loggerFactory);
}
public AzureQueueAdapter(HashRingBasedStreamQueueMapper streamQueueMapper, string dataConnectionString, string deploymentId, string providerName, TimeSpan?messageVisibilityTimeout = null)
{
if (String.IsNullOrEmpty(dataConnectionString))
{
throw new ArgumentNullException("dataConnectionString");
}
if (String.IsNullOrEmpty(deploymentId))
{
throw new ArgumentNullException("deploymentId");
}
DataConnectionString = dataConnectionString;
DeploymentId = deploymentId;
Name = providerName;
MessageVisibilityTimeout = messageVisibilityTimeout;
this.streamQueueMapper = streamQueueMapper;
}
public SQSAdapterFactory(
string name,
SqsOptions sqsOptions,
HashRingStreamQueueMapperOptions queueMapperOptions,
SimpleQueueCacheOptions cacheOptions,
IServiceProvider serviceProvider,
IOptions <ClusterOptions> clusterOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory)
{
this.providerName = name;
this.sqsOptions = sqsOptions;
this.clusterOptions = clusterOptions.Value;
this.serializationManager = serializationManager;
this.loggerFactory = loggerFactory;
streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, this.providerName);
adapterCache = new SimpleQueueAdapterCache(cacheOptions, this.providerName, this.loggerFactory);
}
/// <summary>
/// Async method to delete all used queques, for specific provider and clusterId
/// </summary>
/// <returns> Task object for this async method </returns>
public static async Task DeleteAllUsedQueues(string providerName, string clusterId, string storageConnectionString, ILoggerFactory loggerFactory)
{
if (clusterId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions(), providerName);
List <QueueId> allQueues = queueMapper.GetAllQueues().ToList();
var deleteTasks = new List <Task>();
foreach (var queueId in allQueues)
{
var manager = new SQSStorage(loggerFactory, queueId.ToString(), storageConnectionString, clusterId);
manager.InitQueueAsync().Wait();
deleteTasks.Add(manager.DeleteQueue());
}
await Task.WhenAll(deleteTasks);
}
}
public static async Task DeleteAllUsedQueues(string providerName, string deploymentId, string storageConnectionString)
{
if (deploymentId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(SQSAdapterFactory.NumQueuesDefaultValue, providerName);
List <QueueId> allQueues = queueMapper.GetAllQueues().ToList();
var deleteTasks = new List <Task>();
foreach (var queueId in allQueues)
{
var manager = new SQSStorage(queueId.ToString(), storageConnectionString, deploymentId);
manager.InitQueueAsync().Wait();
deleteTasks.Add(manager.DeleteQueue());
}
await Task.WhenAll(deleteTasks);
}
}
/// <summary>
/// Helper method for testing.
/// </summary>
/// <param name="providerName"></param>
/// <param name="deploymentId"></param>
/// <param name="storageConnectionString"></param>
/// <param name="logger"></param>
/// <returns></returns>
public static async Task DeleteAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString, Logger logger)
{
if (deploymentId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterFactory.DEFAULT_NUM_QUEUES, providerName);
List <QueueId> allQueues = queueMapper.GetAllQueues().ToList();
if (logger != null)
{
logger.Info("About to delete all {0} Stream Queues\n", allQueues.Count);
}
foreach (var queueId in allQueues)
{
var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString);
await manager.DeleteQueue();
}
}
}
/// <summary>
/// Helper method for testing. Clears all messages in all the queues used by the specifed stream provider.
/// </summary>
/// <param name="loggerFactory">logger factory to use</param>
/// <param name="providerName">The Azure Queue stream privider name.</param>
/// <param name="deploymentId">The deployment ID hosting the stream provider.</param>
/// <param name="storageConnectionString">The azure storage connection string.</param>
public static async Task ClearAllUsedAzureQueues(ILoggerFactory loggerFactory, string providerName, string deploymentId, string storageConnectionString)
{
if (deploymentId != null)
{
// TODO: Do not assume defaults !? - jbragg
var queueMapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions(), providerName);
List <QueueId> allQueues = queueMapper.GetAllQueues().ToList();
var deleteTasks = new List <Task>();
foreach (var queueId in allQueues)
{
var manager = new AzureQueueDataManager(loggerFactory, queueId.ToString(), deploymentId, storageConnectionString);
deleteTasks.Add(manager.ClearQueue());
}
await Task.WhenAll(deleteTasks);
}
}
public RabbitMQAdapterFactory(
string name,
RabbitMQStreamProviderOptions options,
HashRingStreamQueueMapperOptions queueMapperOptions,
SimpleQueueCacheOptions cacheOptions,
IServiceProvider serviceProvider,
IOptions <ClusterOptions> clusterOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory)
{
providerName = name;
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.clusterOptions = clusterOptions.Value;
//this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheOptions, providerName, this.loggerFactory);
mapper = ActivatorUtilities.GetServiceOrCreateInstance <TMapper>(serviceProvider);
}
public KafkaQueueAdapter(
HashRingBasedStreamQueueMapper queueMapper,
KafkaStreamProviderOptions options,
string providerName,
ILoggerFactory loggerFactory)
{
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
_options = options ?? throw new ArgumentNullException(nameof(options));
_streamQueueMapper = queueMapper ?? throw new ArgumentNullException(nameof(queueMapper));
Name = providerName;
_loggerFactory = loggerFactory;
_logger = loggerFactory.CreateLogger <KafkaQueueAdapter>();
_producer = new KafkaMessageSender(loggerFactory, options);
_logger.LogInformation("{0} - Created", nameof(KafkaQueueAdapter));
}
