public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter)
{
IStreamQueueBalancer queueBalancer = CreateQueueBalancer(streamProviderName);
var managerId = SystemTargetGrainId.Create(Constants.StreamPullingAgentManagerType, this.siloDetails.SiloAddress, streamProviderName);
var pubsubOptions = this.ServiceProvider.GetOptionsByName <StreamPubSubOptions>(streamProviderName);
var pullingAgentOptions = this.ServiceProvider.GetOptionsByName <StreamPullingAgentOptions>(streamProviderName);
var filter = this.ServiceProvider.GetServiceByName <IStreamFilter>(streamProviderName) ?? new NoOpStreamFilter();
var manager = new PersistentStreamPullingManager(
managerId,
streamProviderName,
this.PubSub(pubsubOptions.PubSubType),
adapterFactory,
queueBalancer,
filter,
pullingAgentOptions,
this.loggerFactory,
this.siloDetails.SiloAddress,
queueAdapter);
var catalog = this.ServiceProvider.GetRequiredService <Catalog>();
catalog.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize();
return(pullingAgentManager);
}
public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter)
{
IStreamQueueBalancer queueBalancer = CreateQueueBalancer(streamProviderName);
var managerId = LegacyGrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var pubsubOptions = this.ServiceProvider.GetOptionsByName <StreamPubSubOptions>(streamProviderName);
var pullingAgentOptions = this.ServiceProvider.GetOptionsByName <StreamPullingAgentOptions>(streamProviderName);
var manager = new PersistentStreamPullingManager(
managerId,
streamProviderName,
this,
this.PubSub(pubsubOptions.PubSubType),
adapterFactory,
queueBalancer,
pullingAgentOptions,
this.loggerFactory,
this.siloDetails.SiloAddress);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize(queueAdapter.AsImmutable());
return(pullingAgentManager);
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (!stateManager.PresetState(ProviderState.Initialized))
{
return;
}
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
adapterFactory = new TAdapterFactory();
// Temporary change, but we need GrainFactory inside ServiceProvider for now,
// so will change it back as soon as we have an action item to add GrainFactory to ServiceProvider.
adapterFactory.Init(config, Name, logger, new GrainFactoryServiceProvider(providerRuntime));
queueAdapter = await adapterFactory.CreateAdapter();
myConfig = new PersistentStreamProviderConfig(config);
this.providerConfig = config;
this.serializationManager = this.providerRuntime.ServiceProvider.GetRequiredService <SerializationManager>();
this.runtimeClient = this.providerRuntime.ServiceProvider.GetRequiredService <IRuntimeClient>();
if (this.myConfig.PubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit ||
this.myConfig.PubSubType == StreamPubSubType.ExplicitGrainBasedOnly)
{
this.streamSubscriptionManager = this.providerRuntime.ServiceProvider
.GetService <IStreamSubscriptionManagerAdmin>().GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
}
string startup;
if (config.Properties.TryGetValue(StartupStatePropertyName, out startup))
{
if (!Enum.TryParse(startup, true, out startupState))
{
throw new ArgumentException(
String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, StartupStatePropertyName, config.Name));
}
}
else
{
startupState = StartupStateDefaultValue;
}
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, Adapter {2} and config {3}, {4} = {5}.",
typeof(TAdapterFactory).Name,
Name,
queueAdapter.Name,
myConfig,
StartupStatePropertyName, startupState);
stateManager.CommitState();
}
public SimpleQueueAdapterCache(IQueueAdapterFactory factory, int cacheSize, Logger logger)
{
if (cacheSize <= 0)
throw new ArgumentOutOfRangeException("cacheSize", "CacheSize must be a positive number.");
this.cacheSize = cacheSize;
this.logger = logger;
caches = new ConcurrentDictionary<QueueId, IQueueCache>();
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
adapterFactory = new TAdapterFactory();
adapterFactory.Init(config, Name, logger);
queueAdapter = await adapterFactory.CreateAdapter();
string timePeriod;
if (!config.Properties.TryGetValue(GET_QUEUE_MESSAGES_TIMER_PERIOD, out timePeriod))
{
getQueueMsgsTimerPeriod = DEFAULT_GET_QUEUE_MESSAGES_TIMER_PERIOD;
}
else
{
getQueueMsgsTimerPeriod = ConfigUtilities.ParseTimeSpan(timePeriod,
"Invalid time value for the " + GET_QUEUE_MESSAGES_TIMER_PERIOD + " property in the provider config values.");
}
string timeout;
if (!config.Properties.TryGetValue(INIT_QUEUE_TIMEOUT, out timeout))
{
initQueueTimeout = DEFAULT_INIT_QUEUE_TIMEOUT;
}
else
{
initQueueTimeout = ConfigUtilities.ParseTimeSpan(timeout,
"Invalid time value for the " + INIT_QUEUE_TIMEOUT + " property in the provider config values.");
}
string balanceTypeString;
balancerType = !config.Properties.TryGetValue(QUEUE_BALANCER_TYPE, out balanceTypeString)
? DEFAULT_STREAM_QUEUE_BALANCER_TYPE
: (StreamQueueBalancerType)Enum.Parse(typeof(StreamQueueBalancerType), balanceTypeString);
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, {2} = {3}, {4} = {5} and with Adapter {6}.",
typeof(TAdapterFactory).Name, Name,
GET_QUEUE_MESSAGES_TIMER_PERIOD, getQueueMsgsTimerPeriod,
INIT_QUEUE_TIMEOUT, this.initQueueTimeout,
queueAdapter.Name);
}
public TimedQueueAdapterCache(IQueueAdapterFactory factory, TimeSpan cacheTimeSpan, int cacheSize, int cacheNumOfBuckets, Logger logger)
{
if (cacheTimeSpan == TimeSpan.Zero)
throw new ArgumentOutOfRangeException(nameof(cacheTimeSpan), "cacheTimeSpan must be larger than zero TimeSpan.");
_cacheTimeSpan = cacheTimeSpan;
Size = cacheSize;
_cacheNumOfBuckets = cacheNumOfBuckets;
_logger = logger;
}
public Task StartPullingAgents(
string streamProviderName,
StreamQueueBalancerType balancerType,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter,
TimeSpan getQueueMsgsTimerPeriod,
TimeSpan initQueueTimeout)
{
return(TaskDone.Done);
}
public SimpleQueueAdapterCache(IQueueAdapterFactory factory, int cacheSize, Logger logger)
{
if (cacheSize <= 0)
{
throw new ArgumentOutOfRangeException("cacheSize", "CacheSize must be a positive number.");
}
this.cacheSize = cacheSize;
this.logger = logger;
caches = new ConcurrentDictionary <QueueId, IQueueCache>();
}
public TimedQueueAdapterCache(IQueueAdapterFactory factory, TimeSpan cacheTimeSpan, int cacheSize, int cacheNumOfBuckets, ILogger logger)
{
if (cacheTimeSpan == TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(nameof(cacheTimeSpan), "cacheTimeSpan must be larger than zero TimeSpan.");
}
_cacheTimeSpan = cacheTimeSpan;
Size = cacheSize;
_cacheNumOfBuckets = cacheNumOfBuckets;
_logger = logger;
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (!stateManager.PresetState(ProviderState.Initialized))
{
return;
}
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
adapterFactory = new TAdapterFactory();
adapterFactory.Init(config, Name, logger, providerRuntime.ServiceProvider);
queueAdapter = await adapterFactory.CreateAdapter();
myConfig = new PersistentStreamProviderConfig(config);
string startup;
if (config.Properties.TryGetValue(StartupStatePropertyName, out startup))
{
if (!Enum.TryParse(startup, true, out startupState))
{
throw new ArgumentException(
String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, StartupStatePropertyName, config.Name));
}
}
else
{
startupState = StartupStateDefaultValue;
}
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, Adapter {2} and config {3}, {4} = {5}.",
typeof(TAdapterFactory).Name,
Name,
queueAdapter.Name,
myConfig,
StartupStatePropertyName, startupState);
stateManager.CommitState();
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
TimeSpan queueGetPeriod,
TimeSpan initQueueTimeout,
TimeSpan maxEvenDeliveryTime)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.queueGetPeriod = queueGetPeriod;
this.initQueueTimeout = initQueueTimeout;
this.maxEvenDeliveryTime = maxEvenDeliveryTime;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = providerRuntime.GetLogger(GetType().Name + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
TimeSpan queueGetPeriod,
TimeSpan initQueueTimeout,
TimeSpan maxEvenDeliveryTime)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.queueGetPeriod = queueGetPeriod;
this.initQueueTimeout = initQueueTimeout;
this.maxEvenDeliveryTime = maxEvenDeliveryTime;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = providerRuntime.GetLogger(GetType().Name + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
internal PersistentStreamPullingManager(
SystemTargetGrainId managerId,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
IStreamFilter streamFilter,
StreamPullingAgentOptions options,
ILoggerFactory loggerFactory,
SiloAddress siloAddress)
: base(managerId, siloAddress, lowPriority: false, loggerFactory)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.options = options;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.streamFilter = streamFilter;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = loggerFactory.CreateLogger($"{GetType().FullName}.{streamProviderName}");
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
this.loggerFactory = loggerFactory;
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
private async Task Init(CancellationToken token)
{
if (!this.stateManager.PresetState(ProviderState.Initialized))
{
return;
}
this.adapterFactory = this.runtime.ServiceProvider.GetRequiredServiceByName <IQueueAdapterFactory>(this.Name);
this.queueAdapter = await adapterFactory.CreateAdapter();
if (this.pubsubOptions.PubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit ||
this.pubsubOptions.PubSubType == StreamPubSubType.ExplicitGrainBasedOnly)
{
this.streamSubscriptionManager = this.runtime.ServiceProvider
.GetService <IStreamSubscriptionManagerAdmin>().GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
}
this.stateManager.CommitState();
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.config = config;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = LogManager.GetLogger(GetType().Name + "-" + streamProviderName, LoggerType.Provider);
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD);
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.config = config;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = TraceLogger.GetLogger(GetType().Name + "-" + streamProviderName, TraceLogger.LoggerType.Provider);
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD);
}
public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter,
PersistentStreamProviderConfig config)
{
IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(config.BalancerType, streamProviderName, this.siloStatusOracle, this.siloDetails.ClusterConfig, this, adapterFactory.GetStreamQueueMapper(), config.SiloMaturityPeriod);
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(config.PubSubType), adapterFactory, queueBalancer, config);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize(queueAdapter.AsImmutable());
return(pullingAgentManager);
}
public async Task StartPullingAgents(
string streamProviderName,
StreamQueueBalancerType balancerType,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter,
TimeSpan getQueueMsgsTimerPeriod,
TimeSpan initQueueTimeout)
{
IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(
balancerType, streamProviderName, Silo.CurrentSilo.LocalSiloStatusOracle, Silo.CurrentSilo.OrleansConfig, this, adapterFactory.GetStreamQueueMapper());
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, adapterFactory, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var managerGrainRef = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await managerGrainRef.Initialize(queueAdapter.AsImmutable());
}
public DictQueueAdapterCache(IQueueAdapterFactory factory, Logger logger)
{
this._logger = logger;
_caches = new ConcurrentDictionary<QueueId, IQueueCache>();
}
private async Task SendAndReceiveFromQueueAdapter(IQueueAdapterFactory adapterFactory, IProviderConfiguration config)
{
IQueueAdapter adapter = await adapterFactory.CreateAdapter();
IQueueAdapterCache cache = adapterFactory.GetQueueAdapterCache();
// Create receiver per queue
IStreamQueueMapper mapper = adapterFactory.GetStreamQueueMapper();
Dictionary <QueueId, IQueueAdapterReceiver> receivers = mapper.GetAllQueues().ToDictionary(queueId => queueId, adapter.CreateReceiver);
Dictionary <QueueId, IQueueCache> caches = mapper.GetAllQueues().ToDictionary(queueId => queueId, cache.CreateQueueCache);
await Task.WhenAll(receivers.Values.Select(receiver => receiver.Initialize(TimeSpan.FromSeconds(5))));
// test using 2 streams
Guid streamId1 = Guid.NewGuid();
Guid streamId2 = Guid.NewGuid();
int receivedBatches = 0;
var streamsPerQueue = new ConcurrentDictionary <QueueId, HashSet <IStreamIdentity> >();
// reader threads (at most 2 active queues because only two streams)
var work = new List <Task>();
foreach (KeyValuePair <QueueId, IQueueAdapterReceiver> receiverKvp in receivers)
{
QueueId queueId = receiverKvp.Key;
var receiver = receiverKvp.Value;
var qCache = caches[queueId];
Task task = Task.Factory.StartNew(() =>
{
while (receivedBatches < NumBatches)
{
var messages = receiver.GetQueueMessagesAsync(CloudQueueMessage.MaxNumberOfMessagesToPeek).Result.ToArray();
if (!messages.Any())
{
continue;
}
foreach (AzureQueueBatchContainer message in messages.Cast <AzureQueueBatchContainer>())
{
streamsPerQueue.AddOrUpdate(queueId,
id => new HashSet <IStreamIdentity> {
new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString())
},
(id, set) =>
{
set.Add(new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString()));
return(set);
});
output.WriteLine("Queue {0} received message on stream {1}", queueId,
message.StreamGuid);
Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <int>().Count()); // "Half the events were ints"
Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <string>().Count()); // "Half the events were strings"
}
Interlocked.Add(ref receivedBatches, messages.Length);
qCache.AddToCache(messages);
}
});
work.Add(task);
}
// send events
List <object> events = CreateEvents(NumMessagesPerBatch);
work.Add(Task.Factory.StartNew(() => Enumerable.Range(0, NumBatches)
.Select(i => i % 2 == 0 ? streamId1 : streamId2)
.ToList()
.ForEach(streamId =>
adapter.QueueMessageBatchAsync(streamId, streamId.ToString(),
events.Take(NumMessagesPerBatch).ToArray(), null, RequestContext.Export()).Wait())));
await Task.WhenAll(work);
// Make sure we got back everything we sent
Assert.Equal(NumBatches, receivedBatches);
// check to see if all the events are in the cache and we can enumerate through them
StreamSequenceToken firstInCache = new EventSequenceToken(0);
foreach (KeyValuePair <QueueId, HashSet <IStreamIdentity> > kvp in streamsPerQueue)
{
var receiver = receivers[kvp.Key];
var qCache = caches[kvp.Key];
foreach (IStreamIdentity streamGuid in kvp.Value)
{
// read all messages in cache for stream
IQueueCacheCursor cursor = qCache.GetCacheCursor(streamGuid, firstInCache);
int messageCount = 0;
StreamSequenceToken tenthInCache = null;
StreamSequenceToken lastToken = firstInCache;
while (cursor.MoveNext())
{
Exception ex;
messageCount++;
IBatchContainer batch = cursor.GetCurrent(out ex);
output.WriteLine("Token: {0}", batch.SequenceToken);
Assert.True(batch.SequenceToken.CompareTo(lastToken) >= 0, $"order check for event {messageCount}");
lastToken = batch.SequenceToken;
if (messageCount == 10)
{
tenthInCache = batch.SequenceToken;
}
}
output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid);
Assert.Equal(NumBatches / 2, messageCount);
Assert.NotNull(tenthInCache);
// read all messages from the 10th
cursor = qCache.GetCacheCursor(streamGuid, tenthInCache);
messageCount = 0;
while (cursor.MoveNext())
{
messageCount++;
}
output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid);
const int expected = NumBatches / 2 - 10 + 1; // all except the first 10, including the 10th (10 + 1)
Assert.Equal(expected, messageCount);
}
}
}
public MySimpleQueueAdapterCache(IQueueAdapterFactory factory, Logger logger)
{
_logger = logger;
// 10 meg buffer pool. 10 1 meg blocks
bufferPool = new FixedSizeObjectPool<FixedSizeBuffer>(10, pool => new FixedSizeBuffer(1 << 20, pool));
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
adapterFactory = new TAdapterFactory();
adapterFactory.Init(config, Name, logger);
queueAdapter = await adapterFactory.CreateAdapter();
string timePeriod;
if (!config.Properties.TryGetValue(GET_QUEUE_MESSAGES_TIMER_PERIOD, out timePeriod))
{
getQueueMsgsTimerPeriod = DEFAULT_GET_QUEUE_MESSAGES_TIMER_PERIOD;
}
else
{
getQueueMsgsTimerPeriod = ConfigUtilities.ParseTimeSpan(timePeriod,
"Invalid time value for the " + GET_QUEUE_MESSAGES_TIMER_PERIOD + " property in the provider config values.");
}
string timeout;
if (!config.Properties.TryGetValue(INIT_QUEUE_TIMEOUT, out timeout))
{
initQueueTimeout = DEFAULT_INIT_QUEUE_TIMEOUT;
}
else
{
initQueueTimeout = ConfigUtilities.ParseTimeSpan(timeout,
"Invalid time value for the " + INIT_QUEUE_TIMEOUT + " property in the provider config values.");
}
string balanceTypeString;
balancerType = !config.Properties.TryGetValue(QUEUE_BALANCER_TYPE, out balanceTypeString)
? DEFAULT_STREAM_QUEUE_BALANCER_TYPE
: (StreamQueueBalancerType)Enum.Parse(typeof(StreamQueueBalancerType), balanceTypeString);
if (!config.Properties.TryGetValue(MAX_EVENT_DELIVERY_TIME, out timeout))
{
maxEventDeliveryTime = DEFAULT_MAX_EVENT_DELIVERY_TIME;
}
else
{
maxEventDeliveryTime = ConfigUtilities.ParseTimeSpan(timeout,
"Invalid time value for the " + MAX_EVENT_DELIVERY_TIME + " property in the provider config values.");
}
string pubSubTypeString;
pubSubType = !config.Properties.TryGetValue(STREAM_PUBSUB_TYPE, out pubSubTypeString)
? DEFAULT_STREAM_PUBSUB_TYPE
: (StreamPubSubType)Enum.Parse(typeof(StreamPubSubType), pubSubTypeString);
string startup;
if (config.Properties.TryGetValue(STARTUP_STATE, out startup))
{
if (!Enum.TryParse(startup, true, out startupState))
{
throw new ArgumentException(
String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, STARTUP_STATE, config.Name));
}
}
else
{
startupState = StreamProviderStartupState.Started;
}
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, {2} = {3}, {4} = {5}, {6} = {7} and with Adapter {8}.",
typeof(TAdapterFactory).Name, Name,
GET_QUEUE_MESSAGES_TIMER_PERIOD, getQueueMsgsTimerPeriod,
INIT_QUEUE_TIMEOUT, initQueueTimeout,
STARTUP_STATE, startupState,
queueAdapter.Name);
}
public MySimpleQueueAdapterCache(IQueueAdapterFactory factory, Logger logger)
{
_logger = logger;
// 10 meg buffer pool. 10 1 meg blocks
bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(10, pool => new FixedSizeBuffer(1 << 20, pool));
}
private async Task SendAndReceiveFromQueueAdapter(IQueueAdapterFactory adapterFactory, IProviderConfiguration config)
{
IQueueAdapter adapter = await adapterFactory.CreateAdapter();
IQueueAdapterCache cache = adapterFactory.GetQueueAdapterCache();
// Create receiver per queue
IStreamQueueMapper mapper = adapterFactory.GetStreamQueueMapper();
Dictionary<QueueId, IQueueAdapterReceiver> receivers = mapper.GetAllQueues().ToDictionary(queueId => queueId, adapter.CreateReceiver);
Dictionary<QueueId, IQueueCache> caches = mapper.GetAllQueues().ToDictionary(queueId => queueId, cache.CreateQueueCache);
await Task.WhenAll(receivers.Values.Select(receiver => receiver.Initialize(TimeSpan.FromSeconds(5))));
// test using 2 streams
Guid streamId1 = Guid.NewGuid();
Guid streamId2 = Guid.NewGuid();
int receivedBatches = 0;
var streamsPerQueue = new ConcurrentDictionary<QueueId, HashSet<IStreamIdentity>>();
// reader threads (at most 2 active queues because only two streams)
var work = new List<Task>();
foreach( KeyValuePair<QueueId, IQueueAdapterReceiver> receiverKvp in receivers)
{
QueueId queueId = receiverKvp.Key;
var receiver = receiverKvp.Value;
var qCache = caches[queueId];
Task task = Task.Factory.StartNew(() =>
{
while (receivedBatches < NumBatches)
{
var messages = receiver.GetQueueMessagesAsync(CloudQueueMessage.MaxNumberOfMessagesToPeek).Result.ToArray();
if (!messages.Any())
{
continue;
}
foreach (AzureQueueBatchContainer message in messages.Cast<AzureQueueBatchContainer>())
{
streamsPerQueue.AddOrUpdate(queueId,
id => new HashSet<IStreamIdentity> { new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString()) },
(id, set) =>
{
set.Add(new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString()));
return set;
});
output.WriteLine("Queue {0} received message on stream {1}", queueId,
message.StreamGuid);
Assert.AreEqual(NumMessagesPerBatch / 2, message.GetEvents<int>().Count(),
"Half the events were ints");
Assert.AreEqual(NumMessagesPerBatch / 2, message.GetEvents<string>().Count(),
"Half the events were strings");
}
Interlocked.Add(ref receivedBatches, messages.Length);
qCache.AddToCache(messages);
}
});
work.Add(task);
}
// send events
List<object> events = CreateEvents(NumMessagesPerBatch);
work.Add(Task.Factory.StartNew(() => Enumerable.Range(0, NumBatches)
.Select(i => i % 2 == 0 ? streamId1 : streamId2)
.ToList()
.ForEach(streamId =>
adapter.QueueMessageBatchAsync(streamId, streamId.ToString(),
events.Take(NumMessagesPerBatch).ToArray(), null, RequestContext.Export()).Wait())));
await Task.WhenAll(work);
// Make sure we got back everything we sent
Assert.AreEqual(NumBatches, receivedBatches);
// check to see if all the events are in the cache and we can enumerate through them
StreamSequenceToken firstInCache = new EventSequenceToken(0);
foreach (KeyValuePair<QueueId, HashSet<IStreamIdentity>> kvp in streamsPerQueue)
{
var receiver = receivers[kvp.Key];
var qCache = caches[kvp.Key];
foreach (IStreamIdentity streamGuid in kvp.Value)
{
// read all messages in cache for stream
IQueueCacheCursor cursor = qCache.GetCacheCursor(streamGuid, firstInCache);
int messageCount = 0;
StreamSequenceToken tenthInCache = null;
StreamSequenceToken lastToken = firstInCache;
while (cursor.MoveNext())
{
Exception ex;
messageCount++;
IBatchContainer batch = cursor.GetCurrent(out ex);
output.WriteLine("Token: {0}", batch.SequenceToken);
Assert.IsTrue(batch.SequenceToken.CompareTo(lastToken) >= 0, "order check for event {0}", messageCount);
lastToken = batch.SequenceToken;
if (messageCount == 10)
{
tenthInCache = batch.SequenceToken;
}
}
output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid);
Assert.AreEqual(NumBatches / 2, messageCount);
Assert.IsNotNull(tenthInCache);
// read all messages from the 10th
cursor = qCache.GetCacheCursor(streamGuid, tenthInCache);
messageCount = 0;
while (cursor.MoveNext())
{
messageCount++;
}
output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid);
const int expected = NumBatches / 2 - 10 + 1; // all except the first 10, including the 10th (10 + 1)
Assert.AreEqual(expected, messageCount);
}
}
}
public QueueAdapterCache(IQueueAdapterFactory factory, TimeSpan timeToKeepMessages, Logger logger)
{
_timeToKeepMessages = timeToKeepMessages;
_logger = logger;
_caches = new ConcurrentDictionary<QueueId, IQueueCache>();
}
