protected async Task MembershipTable_GetGateways()
{
var membershipEntries = Enumerable.Range(0, 10).Select(i => CreateMembershipEntryForTest()).ToArray();
membershipEntries[3].Status = SiloStatus.Active;
membershipEntries[3].ProxyPort = 0;
membershipEntries[5].Status = SiloStatus.Active;
membershipEntries[9].Status = SiloStatus.Active;
var data = await membershipTable.ReadAll();
Assert.NotNull(data);
Assert.Equal(0, data.Members.Count);
var version = data.Version;
foreach (var membershipEntry in membershipEntries)
{
Assert.True(await membershipTable.InsertRow(membershipEntry, version));
version = (await membershipTable.ReadRow(membershipEntry.SiloAddress)).Version;
}
var gateways = await gatewayListProvider.GetGateways();
var entries = new List <string>(gateways.Select(g => g.ToString()));
// only members with a non-zero Gateway port
Assert.DoesNotContain(membershipEntries[3].SiloAddress.ToGatewayUri().ToString(), entries);
// only Active members
Assert.Contains(membershipEntries[5].SiloAddress.ToGatewayUri().ToString(), entries);
Assert.Contains(membershipEntries[9].SiloAddress.ToGatewayUri().ToString(), entries);
Assert.Equal(2, entries.Count);
}
internal async Task RefreshSnapshotLiveGateways_TimerCallback(object context)
{
try
{
if (gatewayListProvider is null)
{
return;
}
// the listProvider.GetGateways() is not under lock.
var allGateways = await gatewayListProvider.GetGateways();
var refreshedGateways = allGateways.Select(gw => gw.ToGatewayAddress()).ToList();
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug("Discovered {GatewayCount} gateways: {Gateways}", refreshedGateways.Count, Utils.EnumerableToString(refreshedGateways));
}
await UpdateLiveGatewaysSnapshot(refreshedGateways, gatewayListProvider.MaxStaleness);
}
catch (Exception exc)
{
logger.Error(ErrorCode.ProxyClient_GetGateways, "Exception occurred during RefreshSnapshotLiveGateways_TimerCallback -> listProvider.GetGateways()", exc);
}
}
protected void Test_GatewaySelection(IGatewayListProvider listProvider)
{
IList <Uri> gatewayUris = listProvider.GetGateways().GetResult();
Assert.True(gatewayUris.Count > 0, $"Found some gateways. Data = {Utils.EnumerableToString(gatewayUris)}");
var gatewayEndpoints = gatewayUris.Select(uri =>
{
return(new IPEndPoint(IPAddress.Parse(uri.Host), uri.Port));
}).ToList();
var cfg = new ClientConfiguration
{
Gateways = gatewayEndpoints
};
var gatewayOptions = new GatewayOptions()
{
GatewayListRefreshPeriod = cfg.GatewayListRefreshPeriod,
PreferedGatewayIndex = cfg.PreferedGatewayIndex
};
var gatewayManager = new GatewayManager(gatewayOptions, listProvider, NullLoggerFactory.Instance);
var counts = new int[4];
for (int i = 0; i < 2300; i++)
{
var ip = gatewayManager.GetLiveGateway();
var addr = IPAddress.Parse(ip.Host);
Assert.Equal(IPAddress.Loopback, addr); // "Incorrect IP address returned for gateway"
Assert.True((0 < ip.Port) && (ip.Port < 5), "Incorrect IP port returned for gateway");
counts[ip.Port - 1]++;
}
// The following needed to be changed as the gateway manager now round-robins through the available gateways, rather than
// selecting randomly based on load numbers.
//Assert.True((500 < counts[0]) && (counts[0] < 1500), "Gateway selection is incorrectly skewed");
//Assert.True((500 < counts[1]) && (counts[1] < 1500), "Gateway selection is incorrectly skewed");
//Assert.True((125 < counts[2]) && (counts[2] < 375), "Gateway selection is incorrectly skewed");
//Assert.True((25 < counts[3]) && (counts[3] < 75), "Gateway selection is incorrectly skewed");
//Assert.True((287 < counts[0]) && (counts[0] < 1150), "Gateway selection is incorrectly skewed");
//Assert.True((287 < counts[1]) && (counts[1] < 1150), "Gateway selection is incorrectly skewed");
//Assert.True((287 < counts[2]) && (counts[2] < 1150), "Gateway selection is incorrectly skewed");
//Assert.True((287 < counts[3]) && (counts[3] < 1150), "Gateway selection is incorrectly skewed");
int low = 2300 / 4;
int up = 2300 / 4;
Assert.True((low <= counts[0]) && (counts[0] <= up), "Gateway selection is incorrectly skewed. " + counts[0]);
Assert.True((low <= counts[1]) && (counts[1] <= up), "Gateway selection is incorrectly skewed. " + counts[1]);
Assert.True((low <= counts[2]) && (counts[2] <= up), "Gateway selection is incorrectly skewed. " + counts[2]);
Assert.True((low <= counts[3]) && (counts[3] <= up), "Gateway selection is incorrectly skewed. " + counts[3]);
}
public GatewayManager(
IOptions <GatewayOptions> gatewayOptions,
IGatewayListProvider gatewayListProvider,
ILoggerFactory loggerFactory,
ConnectionManager connectionManager)
{
this.gatewayOptions = gatewayOptions.Value;
knownDead = new Dictionary <SiloAddress, DateTime>();
rand = new SafeRandom();
logger = loggerFactory.CreateLogger <GatewayManager>();
this.loggerFactory = loggerFactory;
this.connectionManager = connectionManager;
lockable = new object();
gatewayRefreshCallInitiated = false;
ListProvider = gatewayListProvider;
var knownGateways = ListProvider.GetGateways().GetAwaiter().GetResult();
if (knownGateways.Count == 0)
{
string gatewayProviderType = gatewayListProvider.GetType().FullName;
string err = $"Could not find any gateway in {gatewayProviderType}. Orleans client cannot initialize.";
logger.Error(ErrorCode.GatewayManager_NoGateways, err);
throw new OrleansException(err);
}
logger.Info(ErrorCode.GatewayManager_FoundKnownGateways, "Found {0} knownGateways from Gateway listProvider {1}", knownGateways.Count, Utils.EnumerableToString(knownGateways));
if (ListProvider is IGatewayListObservable)
{
((IGatewayListObservable)ListProvider).SubscribeToGatewayNotificationEvents(this);
}
roundRobinCounter = this.gatewayOptions.PreferedGatewayIndex >= 0 ? this.gatewayOptions.PreferedGatewayIndex : rand.Next(knownGateways.Count);
this.knownGateways = cachedLiveGateways = knownGateways.Select(gw => gw.ToGatewayAddress()).ToList();
lastRefreshTime = DateTime.UtcNow;
gatewayRefreshTimer = new AsyncTaskSafeTimer(
this.loggerFactory.CreateLogger <SafeTimer>(),
RefreshSnapshotLiveGateways_TimerCallback,
null,
this.gatewayOptions.GatewayListRefreshPeriod,
this.gatewayOptions.GatewayListRefreshPeriod);
}
internal async Task RefreshSnapshotLiveGateways_TimerCallback(object context)
{
try
{
if (gatewayListProvider is null)
{
return;
}
// the listProvider.GetGateways() is not under lock.
var allGateways = await gatewayListProvider.GetGateways();
var refreshedGateways = allGateways.Select(gw => gw.ToGatewayAddress()).ToList();
await UpdateLiveGatewaysSnapshot(refreshedGateways, gatewayListProvider.MaxStaleness);
}
catch (Exception exc)
{
logger.LogError((int)ErrorCode.ProxyClient_GetGateways, exc, "Exception occurred during RefreshSnapshotLiveGateways_TimerCallback -> listProvider.GetGateways()");
}
}
public GatewayManager(ClientConfiguration cfg, IGatewayListProvider gatewayListProvider, ILoggerFactory loggerFactory)
{
config = cfg;
knownDead = new Dictionary <Uri, DateTime>();
rand = new SafeRandom();
logger = loggerFactory.CreateLogger <GatewayManager>();
this.loggerFactory = loggerFactory;
lockable = new object();
gatewayRefreshCallInitiated = false;
ListProvider = gatewayListProvider;
var knownGateways = ListProvider.GetGateways().GetResult();
if (knownGateways.Count == 0)
{
string gatewayProviderType = gatewayListProvider.GetType().FullName;
string err = String.Format("Could not find any gateway in {0}. Orleans client cannot initialize.", gatewayProviderType);
logger.Error(ErrorCode.GatewayManager_NoGateways, err);
throw new OrleansException(err);
}
logger.Info(ErrorCode.GatewayManager_FoundKnownGateways, "Found {0} knownGateways from Gateway listProvider {1}", knownGateways.Count, Utils.EnumerableToString(knownGateways));
if (ListProvider is IGatewayListObservable)
{
((IGatewayListObservable)ListProvider).SubscribeToGatewayNotificationEvents(this);
}
roundRobinCounter = cfg.PreferedGatewayIndex >= 0 ? cfg.PreferedGatewayIndex : rand.Next(knownGateways.Count);
cachedLiveGateways = knownGateways;
lastRefreshTime = DateTime.UtcNow;
if (ListProvider.IsUpdatable)
{
gatewayRefreshTimer = new SafeTimer(this.loggerFactory.CreateLogger <SafeTimer>(), RefreshSnapshotLiveGateways_TimerCallback, null, config.GatewayListRefreshPeriod, config.GatewayListRefreshPeriod);
}
}
public GatewayManager(ClientConfiguration cfg, IGatewayListProvider gatewayListProvider)
{
config = cfg;
knownDead = new Dictionary<Uri, DateTime>();
rand = new SafeRandom();
logger = TraceLogger.GetLogger("Messaging.GatewayManager", TraceLogger.LoggerType.Runtime);
lockable = new object();
gatewayRefreshCallInitiated = false;
ListProvider = gatewayListProvider;
var knownGateways = ListProvider.GetGateways().GetResult();
if (knownGateways.Count == 0)
{
string gatewayProviderType = gatewayListProvider.GetType().FullName;
string err = String.Format("Could not find any gateway in {0}. Orleans client cannot initialize.", gatewayProviderType);
logger.Error(ErrorCode.GatewayManager_NoGateways, err);
throw new OrleansException(err);
}
logger.Info(ErrorCode.GatewayManager_FoundKnownGateways, "Found {0} knownGateways from Gateway listProvider {1}", knownGateways.Count, Utils.EnumerableToString(knownGateways));
if (ListProvider is IGatewayListObservable)
{
((IGatewayListObservable)ListProvider).SubscribeToGatewayNotificationEvents(this);
}
roundRobinCounter = cfg.PreferedGatewayIndex >= 0 ? cfg.PreferedGatewayIndex : rand.Next(knownGateways.Count);
cachedLiveGateways = knownGateways;
lastRefreshTime = DateTime.UtcNow;
if (ListProvider.IsUpdatable)
{
gatewayRefreshTimer = new SafeTimer(RefreshSnapshotLiveGateways_TimerCallback, null, config.GatewayListRefreshPeriod, config.GatewayListRefreshPeriod);
}
}
internal void RefreshSnapshotLiveGateways_TimerCallback(object context)
{
try
{
if (ListProvider == null || !ListProvider.IsUpdatable)
{
return;
}
// the listProvider.GetGateways() is not under lock.
var currentKnownGateways = ListProvider.GetGateways().GetResult();
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Found {0} knownGateways from Gateway listProvider {1}", currentKnownGateways.Count, Utils.EnumerableToString(currentKnownGateways));
}
// the next one will grab the lock.
UpdateLiveGatewaysSnapshot(currentKnownGateways, ListProvider.MaxStaleness);
}
catch (Exception exc)
{
logger.Error(ErrorCode.ProxyClient_GetGateways, "Exception occurred during RefreshSnapshotLiveGateways_TimerCallback -> listProvider.GetGateways()", exc);
}
}
internal void RefreshSnapshotLiveGateways_TimerCallback(object context)
{
try
{
if (ListProvider == null || !ListProvider.IsUpdatable)
{
return;
}
// the listProvider.GetGateways() is not under lock.
var refreshedGateways = ListProvider.GetGateways().GetResult().Select(gw => gw.ToSiloAddress()).ToList();
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug("Discovered {GatewayCount} gateways: {Gateways}", refreshedGateways.Count, Utils.EnumerableToString(refreshedGateways));
}
// the next one will grab the lock.
UpdateLiveGatewaysSnapshot(refreshedGateways, ListProvider.MaxStaleness);
}
catch (Exception exc)
{
logger.Error(ErrorCode.ProxyClient_GetGateways, "Exception occurred during RefreshSnapshotLiveGateways_TimerCallback -> listProvider.GetGateways()", exc);
}
}
private void Test_GatewaySelection(IGatewayListProvider listProvider)
{
IList<Uri> gatewayUris = listProvider.GetGateways().GetResult();
Assert.IsTrue(gatewayUris.Count > 0, "Found some gateways. Data = {0}", Utils.EnumerableToString(gatewayUris));
var gatewayEndpoints = gatewayUris.Select(uri =>
{
return new IPEndPoint(IPAddress.Parse(uri.Host), uri.Port);
}).ToList();
var cfg = new ClientConfiguration
{
Gateways = gatewayEndpoints
};
var gatewayManager = new GatewayManager(cfg, listProvider);
var counts = new int[4];
for (int i = 0; i < 2300; i++)
{
var ip = gatewayManager.GetLiveGateway();
var addr = IPAddress.Parse(ip.Host);
Assert.AreEqual(IPAddress.Loopback, addr, "Incorrect IP address returned for gateway");
Assert.IsTrue((0 < ip.Port) && (ip.Port < 5), "Incorrect IP port returned for gateway");
counts[ip.Port - 1]++;
}
// The following needed to be changed as the gateway manager now round-robins through the available gateways, rather than
// selecting randomly based on load numbers.
//Assert.IsTrue((500 < counts[0]) && (counts[0] < 1500), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((500 < counts[1]) && (counts[1] < 1500), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((125 < counts[2]) && (counts[2] < 375), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((25 < counts[3]) && (counts[3] < 75), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((287 < counts[0]) && (counts[0] < 1150), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((287 < counts[1]) && (counts[1] < 1150), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((287 < counts[2]) && (counts[2] < 1150), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((287 < counts[3]) && (counts[3] < 1150), "Gateway selection is incorrectly skewed");
int low = 2300 / 4;
int up = 2300 / 4;
Assert.IsTrue((low <= counts[0]) && (counts[0] <= up), "Gateway selection is incorrectly skewed. " + counts[0]);
Assert.IsTrue((low <= counts[1]) && (counts[1] <= up), "Gateway selection is incorrectly skewed. " + counts[1]);
Assert.IsTrue((low <= counts[2]) && (counts[2] <= up), "Gateway selection is incorrectly skewed. " + counts[2]);
Assert.IsTrue((low <= counts[3]) && (counts[3] <= up), "Gateway selection is incorrectly skewed. " + counts[3]);
}