/// <inheritdoc/>
public override Task Initialize(string strProviderName, IStreamQueueMapper queueMapper, TimeSpan siloMaturityPeriod)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
var options = this.serviceProvider.GetRequiredService <IOptionsSnapshot <LeaseBasedQueueBalancerOptions> >().Get(strProviderName);
if (options == null)
{
throw new KeyNotFoundException($"No lease base queue balancer options was configured for provider {strProviderName}, nor was a default configured.");
}
this.leaseProvider = this.serviceProvider.GetRequiredService(options.LeaseProviderType) as ILeaseProvider;
this.leaseLength = options.LeaseLength;
this.allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
this.siloMaturityPeriod = siloMaturityPeriod;
NotifyAfterStart().Ignore();
//make lease renew frequency to be every half of lease time, to avoid renew failing due to timing issues, race condition or clock difference.
var timerLogger = this.loggerFactory.CreateLogger <AsyncTaskSafeTimer>();
this.renewLeaseTimer = new AsyncTaskSafeTimer(timerLogger, this.MaintainAndBalanceQueues, null, this.siloMaturityPeriod, this.leaseLength.Divide(2));
//try to acquire maximum leases every leaseLength
this.tryAcquireMaximumLeaseTimer = new AsyncTaskSafeTimer(timerLogger, this.AcquireLeaseToMeetMaxResponsibilty, null, this.siloMaturityPeriod, this.leaseLength);
//Selector default to round robin selector now, but we can make a further change to make selector configurable if needed. Selector algorithm could
//be affecting queue balancing stablization time in cluster initializing and auto-scaling
this.queueSelector = new RoundRobinSelector <QueueId>(this.allQueues);
return(MaintainAndBalanceQueues(null));
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
IStreamQueueMapper queueMapper,
bool isFixed)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
allQueues = queueMapper.GetAllQueues().ToList();
queueBalanceListeners = new List<IStreamQueueBalanceListener>();
mySiloName = this.siloStatusOracle.SiloName;
this.isFixed = isFixed;
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
}
public async Task Initialize(string strProviderName,
IStreamQueueMapper queueMapper)
{
this.leaseManagerGrain = this.grainFactory.GetGrain <ILeaseManagerGrain>(strProviderName);
await this.leaseManagerGrain.SetQueuesAsLeases(queueMapper.GetAllQueues());
this.id = $"{strProviderName}-{Guid.NewGuid()}";
await GetInitialLease();
}
public override Task Initialize(IStreamQueueMapper queueMapper)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
NotifyAfterStart().Ignore();
return(Task.CompletedTask);
}
public override Task Initialize(IStreamQueueMapper queueMapper)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.allQueues = queueMapper.GetAllQueues().ToList();
NotifyAfterStart().Ignore();
return(base.Initialize(queueMapper));
}
public async Task Initialize(string strProviderName,
IStreamQueueMapper queueMapper,
TimeSpan siloMaturityPeriod,
IProviderConfiguration providerConfig)
{
this.leaseManagerGrain = this.grainFactory.GetGrain <ILeaseManagerGrain>(strProviderName);
await this.leaseManagerGrain.SetQueuesAsLeases(queueMapper.GetAllQueues());
this.id = $"{strProviderName}-{Guid.NewGuid()}";
await GetInitialLease();
}
public override Task Initialize(string strProviderName,
IStreamQueueMapper queueMapper,
TimeSpan siloMaturityPeriod)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
this.siloMaturityPeriod = siloMaturityPeriod;
NotifyAfterStart().Ignore();
return(Task.CompletedTask);
}
/// <summary>
/// Checks to see if deployment configuration has changed, by adding or removing silos.
/// If so, it updates the list of all silo names and creates a new resource balancer.
/// This should occure rarely.
/// </summary>
/// <param name="newSiloNames">new silo names</param>
/// <returns>bool, true if list of all silo names has changed</returns>
private bool UpdateAllSiloNames(IList <string> newSiloNames)
{
// Has configured silo names changed
if (allSiloNames.Count != newSiloNames.Count ||
!allSiloNames.ListEquals(newSiloNames))
{
// record new list of all instance names
allSiloNames = newSiloNames;
// rebuild balancer with new list of instance names
List <QueueId> allQueues = streamQueueMapper.GetAllQueues().ToList();
resourceBalancer = new BestFitBalancer <string, QueueId>(allSiloNames, allQueues);
return(true);
}
return(false);
}
/// <inheritdoc/>
public override Task Initialize(IStreamQueueMapper queueMapper)
{
if (base.Cancellation.IsCancellationRequested)
{
throw new InvalidOperationException("Cannot initialize a terminated balancer.");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
//Selector default to round robin selector now, but we can make a further change to make selector configurable if needed. Selector algorithm could
//be affecting queue balancing stablization time in cluster initializing and auto-scaling
this.queueSelector = new RoundRobinSelector <QueueId>(this.allQueues, new Random(this.GetHashCode()));
return(base.Initialize(queueMapper));
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
IStreamQueueMapper queueMapper)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
streamQueueMapper = queueMapper;
queueBalanceListeners = new List <IStreamQueueBalanceListener>();
mySiloName = this.siloStatusOracle.SiloName;
activeSiloNames = new List <string>();
allSiloNames = this.deploymentConfig.GetAllSiloInstanceNames();
List <QueueId> allQueues = streamQueueMapper.GetAllQueues().ToList();
resourceBalancer = new BestFitBalancer <string, QueueId>(allSiloNames, allQueues);
// get silo names for all active silos
foreach (SiloAddress siloAddress in this.siloStatusOracle.GetApproximateSiloStatuses(true).Keys)
{
string siloName;
if (this.siloStatusOracle.TryGetSiloName(siloAddress, out siloName))
{
activeSiloNames.Add(siloName);
}
}
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
IStreamQueueMapper queueMapper,
TimeSpan maturityPeriod,
bool isFixed)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
queueBalanceListeners = new List <IStreamQueueBalanceListener>();
immatureSilos = new ConcurrentDictionary <SiloAddress, bool>();
this.isFixed = isFixed;
siloMaturityPeriod = maturityPeriod;
isStarting = true;
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
// record all already active silos as already mature.
// Even if they are not yet, they will be mature by the time I mature myself (after I become !isStarting).
foreach (var silo in siloStatusOracle.GetApproximateSiloStatuses(true).Keys.Where(s => !s.Equals(siloStatusOracle.SiloAddress)))
{
immatureSilos[silo] = false; // record as mature
}
NotifyAfterStart().Ignore();
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
IStreamQueueMapper queueMapper,
TimeSpan maturityPeriod,
bool isFixed)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
allQueues = new ReadOnlyCollection<QueueId>(queueMapper.GetAllQueues().ToList());
queueBalanceListeners = new List<IStreamQueueBalanceListener>();
immatureSilos = new ConcurrentDictionary<SiloAddress, bool>();
this.isFixed = isFixed;
siloMaturityPeriod = maturityPeriod;
isStarting = true;
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
// record all already active silos as already mature.
// Even if they are not yet, they will be mature by the time I mature myself (after I become !isStarting).
foreach (var silo in siloStatusOracle.GetApproximateSiloStatuses(true).Keys.Where(s => !s.Equals(siloStatusOracle.SiloAddress)))
{
immatureSilos[silo] = false; // record as mature
}
NotifyAfterStart().Ignore();
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
IStreamQueueMapper queueMapper)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
streamQueueMapper = queueMapper;
queueBalanceListeners = new List<IStreamQueueBalanceListener>();
mySiloName = this.siloStatusOracle.SiloName;
activeSiloNames = new List<string>();
allSiloNames = this.deploymentConfig.GetAllSiloInstanceNames();
List<QueueId> allQueues = streamQueueMapper.GetAllQueues().ToList();
resourceBalancer = new BestFitBalancer<string, QueueId>(allSiloNames, allQueues);
// get silo names for all active silos
foreach (SiloAddress siloAddress in this.siloStatusOracle.GetApproximateSiloStatuses(true).Keys)
{
string siloName;
if (this.siloStatusOracle.TryGetSiloName(siloAddress, out siloName))
{
activeSiloNames.Add(siloName);
}
}
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
}
/// <inheritdoc/>
public override Task Initialize(IStreamQueueMapper queueMapper)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
if (this.allQueues.Count == 0)
{
return(Task.CompletedTask);
}
this.leaseProvider = this.serviceProvider.GetRequiredService(options.LeaseProviderType) as ILeaseProvider;
NotifyAfterStart().Ignore();
//make lease renew frequency to be every half of lease time, to avoid renew failing due to timing issues, race condition or clock difference.
ITimerRegistry timerRegistry = this.serviceProvider.GetRequiredService <ITimerRegistry>();
this.renewLeaseTimer = timerRegistry.RegisterTimer(null, this.MaintainAndBalanceQueues, null, this.options.SiloMaturityPeriod, this.options.LeaseLength.Divide(2));
//try to acquire maximum leases every leaseLength
this.tryAcquireMaximumLeaseTimer = timerRegistry.RegisterTimer(null, this.AcquireLeaseToMeetMaxResponsibility, null, this.options.SiloMaturityPeriod, this.options.SiloMaturityPeriod);
//Selector default to round robin selector now, but we can make a further change to make selector configurable if needed. Selector algorithm could
//be affecting queue balancing stablization time in cluster initializing and auto-scaling
this.queueSelector = new RoundRobinSelector <QueueId>(this.allQueues);
return(MaintainAndBalanceQueues(null));
}
private async Task SendAndReceiveFromQueueAdapter(IQueueAdapterFactory adapterFactory, IProviderConfiguration config)
{
IQueueAdapter adapter = await adapterFactory.CreateAdapter();
IQueueAdapterCache cache = adapterFactory.GetQueueAdapterCache();
// Create receiver per queue
IStreamQueueMapper mapper = adapterFactory.GetStreamQueueMapper();
Dictionary <QueueId, IQueueAdapterReceiver> receivers = mapper.GetAllQueues().ToDictionary(queueId => queueId, adapter.CreateReceiver);
Dictionary <QueueId, IQueueCache> caches = mapper.GetAllQueues().ToDictionary(queueId => queueId, cache.CreateQueueCache);
await Task.WhenAll(receivers.Values.Select(receiver => receiver.Initialize(TimeSpan.FromSeconds(5))));
// test using 2 streams
Guid streamId1 = Guid.NewGuid();
Guid streamId2 = Guid.NewGuid();
int receivedBatches = 0;
var streamsPerQueue = new ConcurrentDictionary <QueueId, HashSet <IStreamIdentity> >();
// reader threads (at most 2 active queues because only two streams)
var work = new List <Task>();
foreach (KeyValuePair <QueueId, IQueueAdapterReceiver> receiverKvp in receivers)
{
QueueId queueId = receiverKvp.Key;
var receiver = receiverKvp.Value;
var qCache = caches[queueId];
Task task = Task.Factory.StartNew(() =>
{
while (receivedBatches < NumBatches)
{
var messages = receiver.GetQueueMessagesAsync(CloudQueueMessage.MaxNumberOfMessagesToPeek).Result.ToArray();
if (!messages.Any())
{
continue;
}
foreach (AzureQueueBatchContainer message in messages.Cast <AzureQueueBatchContainer>())
{
streamsPerQueue.AddOrUpdate(queueId,
id => new HashSet <IStreamIdentity> {
new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString())
},
(id, set) =>
{
set.Add(new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString()));
return(set);
});
output.WriteLine("Queue {0} received message on stream {1}", queueId,
message.StreamGuid);
Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <int>().Count()); // "Half the events were ints"
Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <string>().Count()); // "Half the events were strings"
}
Interlocked.Add(ref receivedBatches, messages.Length);
qCache.AddToCache(messages);
}
});
work.Add(task);
}
// send events
List <object> events = CreateEvents(NumMessagesPerBatch);
work.Add(Task.Factory.StartNew(() => Enumerable.Range(0, NumBatches)
.Select(i => i % 2 == 0 ? streamId1 : streamId2)
.ToList()
.ForEach(streamId =>
adapter.QueueMessageBatchAsync(streamId, streamId.ToString(),
events.Take(NumMessagesPerBatch).ToArray(), null, RequestContext.Export()).Wait())));
await Task.WhenAll(work);
// Make sure we got back everything we sent
Assert.Equal(NumBatches, receivedBatches);
// check to see if all the events are in the cache and we can enumerate through them
StreamSequenceToken firstInCache = new EventSequenceToken(0);
foreach (KeyValuePair <QueueId, HashSet <IStreamIdentity> > kvp in streamsPerQueue)
{
var receiver = receivers[kvp.Key];
var qCache = caches[kvp.Key];
foreach (IStreamIdentity streamGuid in kvp.Value)
{
// read all messages in cache for stream
IQueueCacheCursor cursor = qCache.GetCacheCursor(streamGuid, firstInCache);
int messageCount = 0;
StreamSequenceToken tenthInCache = null;
StreamSequenceToken lastToken = firstInCache;
while (cursor.MoveNext())
{
Exception ex;
messageCount++;
IBatchContainer batch = cursor.GetCurrent(out ex);
output.WriteLine("Token: {0}", batch.SequenceToken);
Assert.True(batch.SequenceToken.CompareTo(lastToken) >= 0, $"order check for event {messageCount}");
lastToken = batch.SequenceToken;
if (messageCount == 10)
{
tenthInCache = batch.SequenceToken;
}
}
output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid);
Assert.Equal(NumBatches / 2, messageCount);
Assert.NotNull(tenthInCache);
// read all messages from the 10th
cursor = qCache.GetCacheCursor(streamGuid, tenthInCache);
messageCount = 0;
while (cursor.MoveNext())
{
messageCount++;
}
output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid);
const int expected = NumBatches / 2 - 10 + 1; // all except the first 10, including the 10th (10 + 1)
Assert.Equal(expected, messageCount);
}
}
}
public async Task Initialize(IStreamQueueMapper queueMapper)
{
await this.leaseManagerGrain.SetQueuesAsLeases(queueMapper.GetAllQueues());
await GetInitialLease();
}