public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
DeploymentBasedQueueBalancerOptions options)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
immatureSilos = new ConcurrentDictionary <SiloAddress, bool>();
this.options = options;
isStarting = true;
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
// record all already active silos as already mature.
// Even if they are not yet, they will be mature by the time I mature myself (after I become !isStarting).
foreach (var silo in siloStatusOracle.GetApproximateSiloStatuses(true).Keys.Where(s => !s.Equals(siloStatusOracle.SiloAddress)))
{
immatureSilos[silo] = false; // record as mature
}
}
public async Task PublishMessageToAllSilos(T message)
{
var tasks = from siloAddress in SiloStatusOracle.GetApproximateSiloStatuses(onlyActive: true).Keys
let remoteGrain = GetGrainServiceForSilo(siloAddress)
select CallRemoteSilo(siloAddress, remoteGrain, message);
await Task.WhenAll(tasks);
}
private async Task OnRefreshClusterMapTimer(object _)
{
// Check if we have to refresh
if (!hasToRefreshClusterGrainInterfaceMap)
{
logger.Verbose3("OnRefreshClusterMapTimer: no refresh required");
return;
}
hasToRefreshClusterGrainInterfaceMap = false;
logger.Info("OnRefreshClusterMapTimer: refresh start");
var activeSilos = statusOracle.GetApproximateSiloStatuses(onlyActive: true);
var knownSilosClusterGrainInterfaceMap = grainTypeManager.GrainInterfaceMapsBySilo;
// Build the new map. Always start by himself
var newSilosClusterGrainInterfaceMap = new Dictionary <SiloAddress, GrainInterfaceMap>
{
{ this.Silo, grainTypeManager.GetTypeCodeMap() }
};
var getGrainInterfaceMapTasks = new List <Task <KeyValuePair <SiloAddress, GrainInterfaceMap> > >();
foreach (var siloAddress in activeSilos.Keys)
{
if (siloAddress.Equals(this.Silo))
{
continue;
}
GrainInterfaceMap value;
if (knownSilosClusterGrainInterfaceMap.TryGetValue(siloAddress, out value))
{
logger.Verbose3($"OnRefreshClusterMapTimer: value already found locally for {siloAddress}");
newSilosClusterGrainInterfaceMap[siloAddress] = value;
}
else
{
// Value not found, let's get it
logger.Verbose3($"OnRefreshClusterMapTimer: value not found locally for {siloAddress}");
getGrainInterfaceMapTasks.Add(GetTargetSiloGrainInterfaceMap(siloAddress));
}
}
if (getGrainInterfaceMapTasks.Any())
{
foreach (var keyValuePair in await Task.WhenAll(getGrainInterfaceMapTasks))
{
if (keyValuePair.Value != null)
{
newSilosClusterGrainInterfaceMap.Add(keyValuePair.Key, keyValuePair.Value);
}
}
}
grainTypeManager.SetInterfaceMapsBySilo(newSilosClusterGrainInterfaceMap);
versionSelectorManager.ResetCache();
}
public Task <SiloDetails[]> GetSiloDetails()
{
// todo this could be improved by using a ISiloStatusListener
// and caching / projecting the changes instead of polling
// should reduce allocations of array's etc
return(Task.FromResult(siloStatusOracle.GetApproximateSiloStatuses(true)
.Select(x => new SiloDetails
{
Status = x.Value.ToString(),
SiloStatus = x.Value,
SiloAddress = x.Key.ToParsableString(),
SiloName = x.Key.ToParsableString() //use the address for naming
})
.ToArray()));
}
private static List <string> GetActiveSilos(ISiloStatusOracle siloStatusOracle, ConcurrentDictionary <SiloAddress, bool> immatureSilos)
{
var activeSiloNames = new List <string>();
foreach (var kvp in siloStatusOracle.GetApproximateSiloStatuses(true))
{
bool immatureBit;
if (!(immatureSilos.TryGetValue(kvp.Key, out immatureBit) && immatureBit)) // if not immature now or any more
{
string siloName;
if (siloStatusOracle.TryGetSiloName(kvp.Key, out siloName))
{
activeSiloNames.Add(siloName);
}
}
}
return(activeSiloNames);
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
IStreamQueueMapper queueMapper,
TimeSpan maturityPeriod,
bool isFixed)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
queueBalanceListeners = new List <IStreamQueueBalanceListener>();
immatureSilos = new ConcurrentDictionary <SiloAddress, bool>();
this.isFixed = isFixed;
siloMaturityPeriod = maturityPeriod;
isStarting = true;
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
// record all already active silos as already mature.
// Even if they are not yet, they will be mature by the time I mature myself (after I become !isStarting).
foreach (var silo in siloStatusOracle.GetApproximateSiloStatuses(true).Keys.Where(s => !s.Equals(siloStatusOracle.SiloAddress)))
{
immatureSilos[silo] = false; // record as mature
}
NotifyAfterStart().Ignore();
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
IStreamQueueMapper queueMapper,
TimeSpan maturityPeriod,
bool isFixed)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
allQueues = new ReadOnlyCollection<QueueId>(queueMapper.GetAllQueues().ToList());
queueBalanceListeners = new List<IStreamQueueBalanceListener>();
immatureSilos = new ConcurrentDictionary<SiloAddress, bool>();
this.isFixed = isFixed;
siloMaturityPeriod = maturityPeriod;
isStarting = true;
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
// record all already active silos as already mature.
// Even if they are not yet, they will be mature by the time I mature myself (after I become !isStarting).
foreach (var silo in siloStatusOracle.GetApproximateSiloStatuses(true).Keys.Where(s => !s.Equals(siloStatusOracle.SiloAddress)))
{
immatureSilos[silo] = false; // record as mature
}
NotifyAfterStart().Ignore();
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
DeploymentBasedQueueBalancerOptions options,
IServiceProvider services,
ILogger <DeploymentBasedQueueBalancer> logger)
: base(services, logger)
{
this.siloStatusOracle = siloStatusOracle ?? throw new ArgumentNullException(nameof(siloStatusOracle));
this.deploymentConfig = deploymentConfig ?? throw new ArgumentNullException(nameof(deploymentConfig));
this.options = options;
isStarting = true;
// record all already active silos as already mature.
// Even if they are not yet, they will be mature by the time I mature myself (after I become !isStarting).
immatureSilos = new ConcurrentDictionary <SiloAddress, bool>(
from s in siloStatusOracle.GetApproximateSiloStatuses(true).Keys
where !s.Equals(siloStatusOracle.SiloAddress)
select new KeyValuePair <SiloAddress, bool>(s, false));
}
private async Task SiloStatusChangeNotification()
{
List <Task> tasks = new List <Task>();
// look at all currently active silos not including myself
foreach (var silo in siloStatusOracle.GetApproximateSiloStatuses(true).Keys.Where(s => !s.Equals(siloStatusOracle.SiloAddress)))
{
bool ignore;
if (!immatureSilos.TryGetValue(silo, out ignore))
{
tasks.Add(RecordImmatureSilo(silo));
}
}
if (!isStarting)
{
// notify, uncoditionaly, and deal with changes in GetMyQueues()
NotifyListeners().Ignore();
}
if (tasks.Count > 0)
{
await Task.WhenAll(tasks);
await NotifyListeners(); // notify, uncoditionaly, and deal with changes it in GetMyQueues()
}
}
private async Task OnRefreshClusterMapTimer(object _)
{
// Check if we have to refresh
if (!hasToRefreshClusterGrainInterfaceMap)
{
if (this.logger.IsEnabled(LogLevel.Trace))
{
logger.Trace("OnRefreshClusterMapTimer: no refresh required");
}
return;
}
while (hasToRefreshClusterGrainInterfaceMap)
{
hasToRefreshClusterGrainInterfaceMap = false;
if (this.logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("OnRefreshClusterMapTimer: refresh start");
}
var activeSilos = statusOracle.GetApproximateSiloStatuses(onlyActive: true);
var knownSilosClusterGrainInterfaceMap = grainTypeManager.GrainInterfaceMapsBySilo;
// Build the new map. Always start by himself
var newSilosClusterGrainInterfaceMap = new Dictionary <SiloAddress, GrainInterfaceMap>
{
{ this.Silo, grainTypeManager.GetTypeCodeMap() }
};
var getGrainInterfaceMapTasks = new List <Task <KeyValuePair <SiloAddress, GrainInterfaceMap> > >();
foreach (var siloAddress in activeSilos.Keys)
{
if (siloAddress.IsSameLogicalSilo(this.Silo))
{
continue;
}
GrainInterfaceMap value;
if (knownSilosClusterGrainInterfaceMap.TryGetValue(siloAddress, out value))
{
if (this.logger.IsEnabled(LogLevel.Trace))
{
this.logger.Trace("OnRefreshClusterMapTimer: value already found locally for {SiloAddress}", siloAddress);
}
newSilosClusterGrainInterfaceMap[siloAddress] = value;
}
else
{
// Value not found, let's get it
if (this.logger.IsEnabled(LogLevel.Debug))
{
this.logger.Debug("OnRefreshClusterMapTimer: value not found locally for {SiloAddress}", siloAddress);
}
getGrainInterfaceMapTasks.Add(GetTargetSiloGrainInterfaceMap(siloAddress));
}
}
if (getGrainInterfaceMapTasks.Any())
{
foreach (var keyValuePair in await Task.WhenAll(getGrainInterfaceMapTasks))
{
if (keyValuePair.Value != null)
{
newSilosClusterGrainInterfaceMap.Add(keyValuePair.Key, keyValuePair.Value);
}
}
}
grainTypeManager.SetInterfaceMapsBySilo(newSilosClusterGrainInterfaceMap);
if (this.versionStore.IsEnabled)
{
await this.GetAndSetDefaultCompatibilityStrategy();
foreach (var kvp in await GetStoredCompatibilityStrategies())
{
this.versionSelectorManager.CompatibilityDirectorManager.SetStrategy(kvp.Key, kvp.Value);
}
await this.GetAndSetDefaultSelectorStrategy();
foreach (var kvp in await GetSelectorStrategies())
{
this.versionSelectorManager.VersionSelectorManager.SetSelector(kvp.Key, kvp.Value);
}
}
versionSelectorManager.ResetCache();
// Either a new silo joined or a refresh failed, so continue until no refresh is required.
if (hasToRefreshClusterGrainInterfaceMap)
{
if (this.logger.IsEnabled(LogLevel.Debug))
{
this.logger.LogDebug("OnRefreshClusterMapTimer: cluster type map still requires a refresh and will be refreshed again after a short delay");
}
await Task.Delay(TimeSpan.FromSeconds(1));
}
}
}
private static int GetActiveSiloCount(ISiloStatusOracle siloStatusOracle)
{
return(siloStatusOracle.GetApproximateSiloStatuses(true).Count);
}
private static List<string> GetActiveSilos(ISiloStatusOracle siloStatusOracle, ConcurrentDictionary<SiloAddress, bool> immatureSilos)
{
var activeSiloNames = new List<string>();
foreach (var kvp in siloStatusOracle.GetApproximateSiloStatuses(true))
{
bool immatureBit;
if (!(immatureSilos.TryGetValue(kvp.Key, out immatureBit) && immatureBit)) // if not immature now or any more
{
string siloName;
if (siloStatusOracle.TryGetSiloName(kvp.Key, out siloName))
{
activeSiloNames.Add(siloName);
}
}
}
return activeSiloNames;
}
