public LocalGrainDirectory(Silo silo)
{
log = TraceLogger.GetLogger("Orleans.GrainDirectory.LocalGrainDirectory");
MyAddress = silo.LocalMessageCenter.MyAddress;
Scheduler = silo.LocalScheduler;
membershipRingList = new List <SiloAddress>();
membershipCache = new HashSet <SiloAddress>();
silo.OrleansConfig.OnConfigChange("Globals/Caching", () =>
{
lock (membershipCache)
{
DirectoryCache = GrainDirectoryCacheFactory <List <Tuple <SiloAddress, ActivationId> > > .CreateGrainDirectoryCache(silo.GlobalConfig);
}
});
maintainer = GrainDirectoryCacheFactory <List <Tuple <SiloAddress, ActivationId> > > .CreateGrainDirectoryCacheMaintainer(this, DirectoryCache);
if (silo.GlobalConfig.SeedNodes.Count > 0)
{
seed = silo.GlobalConfig.SeedNodes.Contains(MyAddress.Endpoint) ? MyAddress : SiloAddress.New(silo.GlobalConfig.SeedNodes[0], 0);
}
stopPreparationResolver = new TaskCompletionSource <bool>();
DirectoryPartition = new GrainDirectoryPartition();
HandoffManager = new GrainDirectoryHandoffManager(this, silo.GlobalConfig);
RemGrainDirectory = new RemoteGrainDirectory(this, Constants.DirectoryServiceId);
CacheValidator = new RemoteGrainDirectory(this, Constants.DirectoryCacheValidatorId);
// add myself to the list of members
AddServer(MyAddress);
Func <SiloAddress, string> siloAddressPrint = (SiloAddress addr) =>
String.Format("{0}/{1:X}", addr.ToLongString(), addr.GetConsistentHashCode());
localLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCAL_ISSUED);
localSuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCAL_SUCCESSES);
fullLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_FULL_ISSUED);
RemoteLookupsSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_REMOTE_SENT);
RemoteLookupsReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_REMOTE_RECEIVED);
LocalDirectoryLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCALDIRECTORY_ISSUED);
LocalDirectorySuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCALDIRECTORY_SUCCESSES);
cacheLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_ISSUED);
cacheSuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_SUCCESSES);
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_HITRATIO, () =>
{
long delta1, delta2;
long curr1 = cacheSuccesses.GetCurrentValueAndDelta(out delta1);
long curr2 = cacheLookups.GetCurrentValueAndDelta(out delta2);
return(String.Format("{0}, Delta={1}",
(curr2 != 0 ? (float)curr1 / (float)curr2 : 0)
, (delta2 != 0 ? (float)delta1 / (float)delta2 : 0)));
});
CacheValidationsSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_VALIDATIONS_CACHE_SENT);
CacheValidationsReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_VALIDATIONS_CACHE_RECEIVED);
registrationsIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_ISSUED);
RegistrationsLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_LOCAL);
RegistrationsRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_REMOTE_SENT);
RegistrationsRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_REMOTE_RECEIVED);
registrationsSingleActIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_ISSUED);
RegistrationsSingleActLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL);
RegistrationsSingleActRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT);
RegistrationsSingleActRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED);
unregistrationsIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_ISSUED);
UnregistrationsLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_LOCAL);
UnregistrationsRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_REMOTE_SENT);
UnregistrationsRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_REMOTE_RECEIVED);
unregistrationsManyIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_ISSUED);
UnregistrationsManyRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_SENT);
UnregistrationsManyRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_RECEIVED);
directoryPartitionCount = IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_PARTITION_SIZE, () => DirectoryPartition.Count);
IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_RINGDISTANCE, () => RingDistanceToSuccessor());
FloatValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_RINGPERCENTAGE, () => (((float)RingDistanceToSuccessor()) / ((float)(int.MaxValue * 2L))) * 100);
FloatValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_AVERAGERINGPERCENTAGE, () => membershipRingList.Count == 0 ? 0 : ((float)100 / (float)membershipRingList.Count));
IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_RINGSIZE, () => membershipRingList.Count);
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING, () =>
{
lock (membershipCache)
{
return(Utils.EnumerableToString(membershipRingList, siloAddressPrint));
}
});
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_PREDECESSORS, () => Utils.EnumerableToString(FindPredecessors(MyAddress, 1), siloAddressPrint));
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_SUCCESSORS, () => Utils.EnumerableToString(FindSuccessors(MyAddress, 1), siloAddressPrint));
}
private static SiloAddress NewSiloAddress(int generation) =>
SiloAddress.New(new IPEndPoint(IPAddress.Parse("127.0.0.1"), 33333), generation);
public LocalGrainDirectory(
IServiceProvider serviceProvider,
ILocalSiloDetails siloDetails,
ISiloStatusOracle siloStatusOracle,
IInternalGrainFactory grainFactory,
Factory <GrainDirectoryPartition> grainDirectoryPartitionFactory,
IOptions <DevelopmentClusterMembershipOptions> developmentClusterMembershipOptions,
IOptions <GrainDirectoryOptions> grainDirectoryOptions,
ILoggerFactory loggerFactory)
{
this.log = loggerFactory.CreateLogger <LocalGrainDirectory>();
var clusterId = siloDetails.ClusterId;
MyAddress = siloDetails.SiloAddress;
this.siloStatusOracle = siloStatusOracle;
this.grainFactory = grainFactory;
ClusterId = clusterId;
DirectoryCache = GrainDirectoryCacheFactory.CreateGrainDirectoryCache(serviceProvider, grainDirectoryOptions.Value);
maintainer =
GrainDirectoryCacheFactory.CreateGrainDirectoryCacheMaintainer(
this,
this.DirectoryCache,
grainFactory,
loggerFactory);
var primarySiloEndPoint = developmentClusterMembershipOptions.Value.PrimarySiloEndpoint;
if (primarySiloEndPoint != null)
{
this.seed = this.MyAddress.Endpoint.Equals(primarySiloEndPoint) ? this.MyAddress : SiloAddress.New(primarySiloEndPoint, 0);
}
DirectoryPartition = grainDirectoryPartitionFactory();
HandoffManager = new GrainDirectoryHandoffManager(this, siloStatusOracle, grainFactory, grainDirectoryPartitionFactory, loggerFactory);
RemoteGrainDirectory = new RemoteGrainDirectory(this, Constants.DirectoryServiceType, loggerFactory);
CacheValidator = new RemoteGrainDirectory(this, Constants.DirectoryCacheValidatorType, loggerFactory);
// add myself to the list of members
AddServer(MyAddress);
Func <SiloAddress, string> siloAddressPrint = (SiloAddress addr) =>
String.Format("{0}/{1:X}", addr.ToLongString(), addr.GetConsistentHashCode());
localLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCAL_ISSUED);
localSuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCAL_SUCCESSES);
fullLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_FULL_ISSUED);
RemoteLookupsSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_REMOTE_SENT);
RemoteLookupsReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_REMOTE_RECEIVED);
LocalDirectoryLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCALDIRECTORY_ISSUED);
LocalDirectorySuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCALDIRECTORY_SUCCESSES);
cacheLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_ISSUED);
cacheSuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_SUCCESSES);
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_HITRATIO, () =>
{
long delta1, delta2;
long curr1 = cacheSuccesses.GetCurrentValueAndDelta(out delta1);
long curr2 = cacheLookups.GetCurrentValueAndDelta(out delta2);
return(String.Format("{0}, Delta={1}",
(curr2 != 0 ? (float)curr1 / (float)curr2 : 0)
, (delta2 != 0 ? (float)delta1 / (float)delta2 : 0)));
});
CacheValidationsSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_VALIDATIONS_CACHE_SENT);
CacheValidationsReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_VALIDATIONS_CACHE_RECEIVED);
registrationsIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_ISSUED);
RegistrationsLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_LOCAL);
RegistrationsRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_REMOTE_SENT);
RegistrationsRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_REMOTE_RECEIVED);
registrationsSingleActIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_ISSUED);
RegistrationsSingleActLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL);
RegistrationsSingleActRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT);
RegistrationsSingleActRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED);
unregistrationsIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_ISSUED);
UnregistrationsLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_LOCAL);
UnregistrationsRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_REMOTE_SENT);
UnregistrationsRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_REMOTE_RECEIVED);
unregistrationsManyIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_ISSUED);
UnregistrationsManyRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_SENT);
UnregistrationsManyRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_RECEIVED);
directoryPartitionCount = IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_PARTITION_SIZE, () => DirectoryPartition.Count);
IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_RINGDISTANCE, () => RingDistanceToSuccessor());
FloatValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_RINGPERCENTAGE, () => (((float)this.RingDistanceToSuccessor()) / ((float)(int.MaxValue * 2L))) * 100);
FloatValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_AVERAGERINGPERCENTAGE, () =>
{
var ring = this.directoryMembership.MembershipRingList;
return(ring.Count == 0 ? 0 : ((float)100 / (float)ring.Count));
});
IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_RINGSIZE, () => this.directoryMembership.MembershipRingList.Count);
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING, () =>
{
var ring = this.directoryMembership.MembershipRingList;
return(Utils.EnumerableToString(ring, siloAddressPrint));
});
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_PREDECESSORS, () => Utils.EnumerableToString(this.FindPredecessors(this.MyAddress, 1), siloAddressPrint));
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_SUCCESSORS, () => Utils.EnumerableToString(this.FindSuccessors(this.MyAddress, 1), siloAddressPrint));
}
private async Task <IMembershipTableSystemTarget> GetMembershipTable()
{
var options = this.serviceProvider.GetRequiredService <IOptions <DevelopmentClusterMembershipOptions> >().Value;
if (options.PrimarySiloEndpoint == null)
{
throw new OrleansConfigurationException(
$"{nameof(DevelopmentClusterMembershipOptions)}.{nameof(options.PrimarySiloEndpoint)} must be set when using development clustering.");
}
var siloDetails = this.serviceProvider.GetService <ILocalSiloDetails>();
bool isPrimarySilo = siloDetails.SiloAddress.Endpoint.Equals(options.PrimarySiloEndpoint);
if (isPrimarySilo)
{
this.logger.Info(ErrorCode.MembershipFactory1, "Creating in-memory membership table");
var catalog = serviceProvider.GetRequiredService <Catalog>();
catalog.RegisterSystemTarget(ActivatorUtilities.CreateInstance <MembershipTableSystemTarget>(serviceProvider));
}
var grainFactory = this.serviceProvider.GetRequiredService <IInternalGrainFactory>();
var result = grainFactory.GetSystemTarget <IMembershipTableSystemTarget>(Constants.SystemMembershipTableType, SiloAddress.New(options.PrimarySiloEndpoint, 0));
if (isPrimarySilo)
{
await this.WaitForTableGrainToInit(result);
}
return(result);
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
var stored = (StoredSiloAddress)serializer.Deserialize(reader, typeof(StoredSiloAddress));
return(SiloAddress.New(stored.Endpoint, stored.Generation));
}
// This function is called asynchronously from gateway refresh timer.
private void UpdateLiveGatewaysSnapshot(IEnumerable <SiloAddress> refreshedGateways, TimeSpan maxStaleness)
{
// this is a short lock, protecting the access to knownDead and cachedLiveGateways.
lock (lockable)
{
// now take whatever listProvider gave us and exclude those we think are dead.
var live = new List <SiloAddress>();
var now = DateTime.UtcNow;
this.knownGateways = refreshedGateways as List <SiloAddress> ?? refreshedGateways.ToList();
foreach (SiloAddress trial in knownGateways)
{
var address = trial.Generation == 0 ? trial : SiloAddress.New(trial.Endpoint, 0);
// We consider a node to be dead if we recorded it is dead due to socket error
// and it was recorded (diedAt) not too long ago (less than maxStaleness ago).
// The latter is to cover the case when the Gateway provider returns an outdated list that does not yet reflect the actually recently died Gateway.
// If it has passed more than maxStaleness - we assume maxStaleness is the upper bound on Gateway provider freshness.
var isDead = false;
if (knownDead.TryGetValue(address, out var diedAt))
{
if (now.Subtract(diedAt) < maxStaleness)
{
isDead = true;
}
else
{
// Remove stale entries.
knownDead.Remove(address);
}
}
if (!isDead)
{
live.Add(address);
}
}
if (live.Count == 0)
{
logger.Warn(
ErrorCode.GatewayManager_AllGatewaysDead,
"All gateways have previously been marked as dead. Clearing the list of dead gateways to expedite reconnection.");
live.AddRange(knownGateways);
knownDead.Clear();
}
// swap cachedLiveGateways pointer in one atomic operation
cachedLiveGateways = live;
DateTime prevRefresh = lastRefreshTime;
lastRefreshTime = now;
if (logger.IsEnabled(LogLevel.Information))
{
logger.Info(ErrorCode.GatewayManager_FoundKnownGateways,
"Refreshed the live Gateway list. Found {0} gateways from Gateway listProvider: {1}. Picked only known live out of them. Now has {2} live Gateways: {3}. Previous refresh time was = {4}",
knownGateways.Count(),
Utils.EnumerableToString(knownGateways),
cachedLiveGateways.Count,
Utils.EnumerableToString(cachedLiveGateways),
prevRefresh);
}
this.AbortEvictedGatewayConnections(live);
}
}
private static Uri ConvertToGatewayUri(SiloEntity gateway)
{
SiloAddress address = SiloAddress.New(new IPEndPoint(IPAddress.Parse(gateway.Address), gateway.ProxyPort.Value), gateway.Generation);
return(address.ToGatewayUri());
}
private static SiloAddress NewSiloAddress(string ip, ushort port, int generation) => SiloAddress.New(new IPEndPoint(IPAddress.Parse(ip), port), generation);
public async Task UnknownSilosAreAccountedForDuringUpdate()
{
var now = new[] { DateTime.UtcNow };
this.unknownSiloMonitor.GetDateTime = () => now[0];
var listener = new MockStatusListener();
this.oracle.SubscribeToSiloStatusEvents(listener);
await this.oracle.Start();
await this.oracle.BecomeActive();
var silos = new[]
{
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 1), 1),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 2), 1),
"HappyNewSilo"),
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 3), 2),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 4), 2),
"OtherNewSilo"),
};
this.resolver.Notify(silos);
listener.WaitForVersion(4);
Assert.Equal(3, listener.Silos.Count);
// Query for an unknown silo. Doing so should result in the silo being recorded as unknown.
var unknownSilo1 = SiloAddress.New(new IPEndPoint(IPAddress.Parse("1.1.1.1"), 3), 3);
Assert.Equal(SiloStatus.None, this.oracle.GetApproximateSiloStatus(unknownSilo1));
this.resolver.Notify(silos);
// The status should not have changed.
Assert.Equal(SiloStatus.None, this.oracle.GetApproximateSiloStatus(unknownSilo1));
now[0] += this.fabricClusteringOptions.UnknownSiloRemovalPeriod + TimeSpan.FromMilliseconds(1);
this.resolver.Notify(silos);
listener.WaitForVersion(5);
// After the delay, the silo should be declared dead.
Assert.Equal(SiloStatus.Dead, this.oracle.GetApproximateSiloStatus(unknownSilo1));
// Query for another unknown silo using a different mechanism.
var unknownSilo2 = SiloAddress.New(new IPEndPoint(IPAddress.Parse("2.2.2.2"), 4), 4);
Assert.False(this.oracle.IsFunctionalDirectory(unknownSilo2));
// Report a new silo with the same endpoint as the unknown silo but a higher generation.
silos = new[]
{
silos[0],
silos[1],
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Parse("2.2.2.2"), 4), 5),
SiloAddress.Zero,
"TippyTop")
};
this.resolver.Notify(silos);
listener.WaitForVersion(6);
Assert.True(this.oracle.IsDeadSilo(unknownSilo2));
}
private static Uri ReturnGatewayUri(MongoMembershipDocument record)
{
var siloAddress = SiloAddress.FromParsableString(record.SiloAddress);
return(SiloAddress.New(new IPEndPoint(siloAddress.Endpoint.Address, record.ProxyPort), siloAddress.Generation).ToGatewayUri());
}
private static void RunTest(int numItems)
{
InvokeMethodRequest request = new InvokeMethodRequest(0, 0, null);
Message resp = Message.CreateMessage(request, InvokeMethodOptions.None);
resp.Id = new CorrelationId();
resp.SendingSilo = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 200), 0);
resp.TargetSilo = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 300), 0);
resp.SendingGrain = GrainId.NewId();
resp.TargetGrain = GrainId.NewId();
resp.SetHeader(Message.Header.ALWAYS_INTERLEAVE, true);
resp.SetHeader(Message.Header.CACHE_INVALIDATION_HEADER, "TestBar");
//resp.SetStringBody("This is test data");
List <object> requestBody = new List <object>();
for (int k = 0; k < numItems; k++)
{
requestBody.Add(k + ": test line");
}
resp.BodyObject = requestBody;
string s = resp.ToString();
Console.WriteLine(s);
var serialized = resp.Serialize();
int length = serialized.Sum <ArraySegment <byte> >(x => x.Count);
byte[] data = new byte[length];
int n = 0;
foreach (var buffer in serialized)
{
Array.Copy(buffer.Array, buffer.Offset, data, n, buffer.Count);
n += buffer.Count;
}
resp.ReleaseBodyAndHeaderBuffers();
int headerLength = BitConverter.ToInt32(data, 0);
int bodyLength = BitConverter.ToInt32(data, 4);
Assert.AreEqual <int>(length, headerLength + bodyLength + 8, "Serialized lengths are incorrect");
byte[] header = new byte[headerLength];
Array.Copy(data, 8, header, 0, headerLength);
byte[] body = new byte[bodyLength];
Array.Copy(data, 8 + headerLength, body, 0, bodyLength);
var headerList = new List <ArraySegment <byte> >();
headerList.Add(new ArraySegment <byte>(header));
var bodyList = new List <ArraySegment <byte> >();
bodyList.Add(new ArraySegment <byte>(body));
var resp1 = new Message(headerList, bodyList);
//byte[] serialized = resp.FormatForSending();
//Message resp1 = new Message(serialized, serialized.Length);
Assert.AreEqual <Message.Categories>(resp.Category, resp1.Category, "Category is incorrect");
Assert.AreEqual <Message.Directions>(resp.Direction, resp1.Direction, "Direction is incorrect");
Assert.AreEqual <CorrelationId>(resp.Id, resp1.Id, "Correlation ID is incorrect");
Assert.AreEqual <bool>((bool)resp.GetHeader(Message.Header.ALWAYS_INTERLEAVE), (bool)resp1.GetHeader(Message.Header.ALWAYS_INTERLEAVE), "Foo Boolean is incorrect");
Assert.AreEqual <string>((string)resp.GetHeader(Message.Header.CACHE_INVALIDATION_HEADER), (string)resp1.GetHeader(Message.Header.CACHE_INVALIDATION_HEADER), "Bar string is incorrect");
Assert.IsTrue(resp.TargetSilo.Equals(resp1.TargetSilo), "TargetSilo is incorrect");
Assert.IsTrue(resp.SendingSilo.Equals(resp1.SendingSilo), "SendingSilo is incorrect");
Assert.IsInstanceOfType(resp1.BodyObject, typeof(List <object>), "Body object is wrong type");
List <object> responseList = resp1.BodyObject as List <object>;
Assert.AreEqual <int>(numItems, responseList.Count, "Body list has wrong number of entries");
for (int k = 0; k < numItems; k++)
{
Assert.IsInstanceOfType(responseList[k], typeof(string), "Body list item " + k + " has wrong type");
Assert.AreEqual <string>((string)(requestBody[k]), (string)(responseList[k]), "Body list item " + k + " is incorrect");
}
}
private ValueTask<Connection> GetGatewayConnection(Message msg)
{
// If there's a specific gateway specified, use it
if (msg.TargetSilo != null && gatewayManager.GetLiveGateways().Contains(msg.TargetSilo))
{
var siloAddress = SiloAddress.New(msg.TargetSilo.Endpoint, 0);
var connectionTask = this.connectionManager.GetConnection(siloAddress);
if (connectionTask.IsCompletedSuccessfully) return connectionTask;
return ConnectAsync(msg.TargetSilo, connectionTask, msg, directGatewayMessage: true);
}
// For untargeted messages to system targets, and for unordered messages, pick a next connection in round robin fashion.
if (msg.TargetGrain.IsSystemTarget || msg.IsUnordered)
{
// Get the cached list of live gateways.
// Pick a next gateway name in a round robin fashion.
// See if we have a live connection to it.
// If Yes, use it.
// If not, create a new GatewayConnection and start it.
// If start fails, we will mark this connection as dead and remove it from the GetCachedLiveGatewayNames.
int msgNumber = Interlocked.Increment(ref numMessages);
var gatewayAddresses = gatewayManager.GetLiveGateways();
int numGateways = gatewayAddresses.Count;
if (numGateways == 0)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend, "Unable to send message {0}; gateway manager state is {1}", msg, gatewayManager);
return new ValueTask<Connection>(default(Connection));
}
var gatewayAddress = gatewayAddresses[msgNumber % numGateways];
var connectionTask = this.connectionManager.GetConnection(gatewayAddress);
if (connectionTask.IsCompletedSuccessfully) return connectionTask;
return ConnectAsync(gatewayAddress, connectionTask, msg, directGatewayMessage: false);
}
// Otherwise, use the buckets to ensure ordering.
var index = msg.TargetGrain.GetHashCode_Modulo((uint)grainBuckets.Length);
// Repeated from above, at the declaration of the grainBuckets array:
// Requests are bucketed by GrainID, so that all requests to a grain get routed through the same bucket.
// Each bucket holds a (possibly null) weak reference to a GatewayConnection object. That connection instance is used
// if the WeakReference is non-null, is alive, and points to a live gateway connection. If any of these conditions is
// false, then a new gateway is selected using the gateway manager, and a new connection established if necessary.
WeakReference<Connection> weakRef = grainBuckets[index];
if (weakRef != null && weakRef.TryGetTarget(out var existingConnection) && existingConnection.IsValid)
{
return new ValueTask<Connection>(existingConnection);
}
var addr = gatewayManager.GetLiveGateway();
if (addr == null)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend_NoGateway, "Unable to send message {0}; gateway manager state is {1}", msg, gatewayManager);
return new ValueTask<Connection>(default(Connection));
}
if (logger.IsEnabled(LogLevel.Trace)) logger.Trace(ErrorCode.ProxyClient_NewBucketIndex, "Starting new bucket index {0} for ordered messages to grain {1}", index, msg.TargetGrain);
if (logger.IsEnabled(LogLevel.Debug)) logger.Debug(
ErrorCode.ProxyClient_CreatedGatewayToGrain,
"Creating gateway to {0} for message to grain {1}, bucket {2}, grain id hash code {3}X",
addr,
msg.TargetGrain,
index,
msg.TargetGrain.GetHashCode().ToString("x"));
var gatewayConnection = this.connectionManager.GetConnection(addr);
if (gatewayConnection.IsCompletedSuccessfully)
{
this.UpdateBucket(index, gatewayConnection.Result);
return gatewayConnection;
}
return AddToBucketAsync(index, gatewayConnection, addr);
async ValueTask<Connection> AddToBucketAsync(
uint bucketIndex,
ValueTask<Connection> connectionTask,
SiloAddress gatewayAddress)
{
try
{
var connection = await connectionTask.ConfigureAwait(false);
this.UpdateBucket(bucketIndex, connection);
return connection;
}
catch
{
this.gatewayManager.MarkAsDead(gatewayAddress);
this.UpdateBucket(bucketIndex, null);
throw;
}
}
async ValueTask<Connection> ConnectAsync(
SiloAddress gateway,
ValueTask<Connection> connectionTask,
Message message,
bool directGatewayMessage)
{
Connection result = default;
try
{
return result = await connectionTask;
}
catch (Exception exception) when (directGatewayMessage)
{
RejectMessage(message, string.Format("Target silo {0} is unavailable", message.TargetSilo), exception);
return null;
}
finally
{
if (result is null) this.gatewayManager.MarkAsDead(gateway);
}
}
}
private SiloAddress GenerateSiloAddress() => SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 5000), ++generation);
public static SiloAddress NewLocalSiloAddress(int gen)
{
return(SiloAddress.New(localEndpoint, gen));
}
private void RunTest(int numItems)
{
InvokeMethodRequest request = new InvokeMethodRequest(0, 2, 0, null);
Message resp = this.messageFactory.CreateMessage(request, InvokeMethodOptions.None);
resp.Id = new CorrelationId();
resp.SendingSilo = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 200), 0);
resp.TargetSilo = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 300), 0);
resp.SendingGrain = GrainId.NewId();
resp.TargetGrain = GrainId.NewId();
resp.IsAlwaysInterleave = true;
Assert.True(resp.IsUsingInterfaceVersions);
List <object> requestBody = new List <object>();
for (int k = 0; k < numItems; k++)
{
requestBody.Add(k + ": test line");
}
resp.BodyObject = requestBody;
string s = resp.ToString();
output.WriteLine(s);
int dummy;
var serialized = resp.Serialize(this.fixture.SerializationManager, out dummy, out dummy);
int length = serialized.Sum <ArraySegment <byte> >(x => x.Count);
byte[] data = new byte[length];
int n = 0;
foreach (var buffer in serialized)
{
Array.Copy(buffer.Array, buffer.Offset, data, n, buffer.Count);
n += buffer.Count;
}
resp.ReleaseBodyAndHeaderBuffers();
var resp1 = DeserializeMessage(length, data);
//byte[] serialized = resp.FormatForSending();
//Message resp1 = new Message(serialized, serialized.Length);
Assert.Equal(resp.Category, resp1.Category); //Category is incorrect"
Assert.Equal(resp.Direction, resp1.Direction); //Direction is incorrect
Assert.Equal(resp.Id, resp1.Id); //Correlation ID is incorrect
Assert.Equal(resp.IsAlwaysInterleave, resp1.IsAlwaysInterleave); //Foo Boolean is incorrect
Assert.Equal(resp.CacheInvalidationHeader, resp1.CacheInvalidationHeader); //Bar string is incorrect
Assert.True(resp.TargetSilo.Equals(resp1.TargetSilo));
Assert.True(resp.TargetGrain.Equals(resp1.TargetGrain));
Assert.True(resp.SendingGrain.Equals(resp1.SendingGrain));
Assert.True(resp.SendingSilo.Equals(resp1.SendingSilo)); //SendingSilo is incorrect
Assert.True(resp1.IsUsingInterfaceVersions);
List <object> responseList = Assert.IsAssignableFrom <List <object> >(resp1.GetDeserializedBody(this.fixture.SerializationManager));
Assert.Equal <int>(numItems, responseList.Count); //Body list has wrong number of entries
for (int k = 0; k < numItems; k++)
{
Assert.IsAssignableFrom <string>(responseList[k]); //Body list item " + k + " has wrong type
Assert.Equal <string>((string)(requestBody[k]), (string)(responseList[k])); //Body list item " + k + " is incorrect
}
}
private static MembershipEntry ConvertFromRow(SqlDataReader results, out string eTag, out int tableVersion, out string versionETag)
{
var entry = new MembershipEntry();
int port = results.GetInt32(PortIdx);
int gen = results.GetInt32(GenerationIdx);
entry.SiloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Parse(results.GetString(AddressIdx)), port), gen);
entry.HostName = results.GetString(HostNameIdx);
entry.Status = (SiloStatus)results.GetInt32(StatusIdx);
if (!results.GetSqlInt32(ProxyPortIdx).IsNull)
{
entry.ProxyPort = results.GetInt32(ProxyPortIdx);
}
if (!results.GetSqlBoolean(PrimaryIdx).IsNull)
{
entry.IsPrimary = results.GetBoolean(PrimaryIdx);
}
entry.RoleName = results.GetString(RoleNameIdx);
entry.InstanceName = results.GetString(InstanceNameIdx);
if (!results.GetSqlInt32(UpdateZoneIdx).IsNull)
{
entry.UpdateZone = results.GetInt32(UpdateZoneIdx);
}
if (!results.GetSqlInt32(FaultZoneIdx).IsNull)
{
entry.FaultZone = results.GetInt32(FaultZoneIdx);
}
if (!results.GetSqlDateTime(StartTimeIdx).IsNull)
{
entry.StartTime = results.GetDateTime(StartTimeIdx);
}
if (!results.GetSqlDateTime(IAmAliveTimeIdx).IsNull)
{
entry.IAmAliveTime = results.GetDateTime(IAmAliveTimeIdx);
}
eTag = results.GetString(ETagIdx);
tableVersion = (int)results.GetInt64(VersionIdx);
versionETag = results.GetString(VersionETagIdx);
var suspectingSilosString = results.GetSqlString(SuspectingSilosIdx);
var suspectingTimesString = results.GetSqlString(SuspectingTimesIdx);
List <SiloAddress> suspectingSilos = new List <SiloAddress>();
List <DateTime> suspectingTimes = new List <DateTime>();
if (!suspectingSilosString.IsNull && !string.IsNullOrEmpty(suspectingSilosString.Value))
{
string[] silos = suspectingSilosString.Value.Split('|');
foreach (string silo in silos)
{
suspectingSilos.Add(SiloAddress.FromParsableString(silo));
}
}
if (!suspectingTimesString.IsNull && !string.IsNullOrEmpty(suspectingTimesString.Value))
{
string[] times = suspectingTimesString.Value.Split('|');
foreach (string time in times)
{
suspectingTimes.Add(TraceLogger.ParseDate(time));
}
}
if (suspectingSilos.Count != suspectingTimes.Count)
{
throw new OrleansException(String.Format("SuspectingSilos.Length of {0} as read from SQL table is not eqaul to SuspectingTimes.Length of {1}", suspectingSilos.Count, suspectingTimes.Count));
}
for (int i = 0; i < suspectingSilos.Count; i++)
{
entry.AddSuspector(suspectingSilos[i], suspectingTimes[i]);
}
return(entry);
}
private static MembershipEntry Parse(SiloInstanceTableEntry tableEntry)
{
var parse = new MembershipEntry
{
HostName = tableEntry.HostName,
Status = (SiloStatus)Enum.Parse(typeof(SiloStatus), tableEntry.Status)
};
if (!string.IsNullOrEmpty(tableEntry.ProxyPort))
{
parse.ProxyPort = int.Parse(tableEntry.ProxyPort);
}
int port = 0;
if (!string.IsNullOrEmpty(tableEntry.Port))
{
int.TryParse(tableEntry.Port, out port);
}
int gen = 0;
if (!string.IsNullOrEmpty(tableEntry.Generation))
{
int.TryParse(tableEntry.Generation, out gen);
}
parse.SiloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Parse(tableEntry.Address), port), gen);
parse.RoleName = tableEntry.RoleName;
parse.InstanceName = tableEntry.InstanceName;
if (!string.IsNullOrEmpty(tableEntry.UpdateZone))
{
parse.UpdateZone = int.Parse(tableEntry.UpdateZone);
}
if (!string.IsNullOrEmpty(tableEntry.FaultZone))
{
parse.FaultZone = int.Parse(tableEntry.FaultZone);
}
parse.StartTime = !string.IsNullOrEmpty(tableEntry.StartTime) ?
TraceLogger.ParseDate(tableEntry.StartTime) : default(DateTime);
parse.IAmAliveTime = !string.IsNullOrEmpty(tableEntry.IAmAliveTime) ?
TraceLogger.ParseDate(tableEntry.IAmAliveTime) : default(DateTime);
var suspectingSilos = new List <SiloAddress>();
var suspectingTimes = new List <DateTime>();
if (!string.IsNullOrEmpty(tableEntry.SuspectingSilos))
{
string[] silos = tableEntry.SuspectingSilos.Split('|');
foreach (string silo in silos)
{
suspectingSilos.Add(SiloAddress.FromParsableString(silo));
}
}
if (!string.IsNullOrEmpty(tableEntry.SuspectingTimes))
{
string[] times = tableEntry.SuspectingTimes.Split('|');
foreach (string time in times)
{
suspectingTimes.Add(TraceLogger.ParseDate(time));
}
}
if (suspectingSilos.Count != suspectingTimes.Count)
{
throw new OrleansException(String.Format("SuspectingSilos.Length of {0} as read from Azure table is not eqaul to SuspectingTimes.Length of {1}", suspectingSilos.Count, suspectingTimes.Count));
}
for (int i = 0; i < suspectingSilos.Count; i++)
{
parse.AddSuspector(suspectingSilos[i], suspectingTimes[i]);
}
return(parse);
}
public void ID_IsSystem()
{
GrainId testGrain = Orleans.Runtime.Constants.DirectoryServiceId;
output.WriteLine("Testing GrainID " + testGrain);
Assert.True(testGrain.IsSystemTarget); // System grain ID is not flagged as a system ID
GrainId sGrain = (GrainId)this.environment.SerializationManager.DeepCopy(testGrain);
output.WriteLine("Testing GrainID " + sGrain);
Assert.True(sGrain.IsSystemTarget); // String round-trip grain ID is not flagged as a system ID
Assert.Equal(testGrain, sGrain); // Should be equivalent GrainId object
Assert.Same(testGrain, sGrain); // Should be same / intern'ed GrainId object
ActivationId testActivation = ActivationId.GetSystemActivation(testGrain, SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 2456), 0));
output.WriteLine("Testing ActivationID " + testActivation);
Assert.True(testActivation.IsSystem); // System activation ID is not flagged as a system ID
}
public void ID_IsSystem()
{
GrainId testGrain = Constants.DirectoryServiceId;
Console.WriteLine("Testing GrainID " + testGrain);
Assert.IsTrue(testGrain.IsSystemTarget, "System grain ID is not flagged as a system ID");
GrainId sGrain = (GrainId)SerializationManager.DeepCopy(testGrain);
Console.WriteLine("Testing GrainID " + sGrain);
Assert.IsTrue(sGrain.IsSystemTarget, "String round-trip grain ID is not flagged as a system ID");
Assert.AreEqual(testGrain, sGrain, "Should be equivalent GrainId object");
Assert.AreSame(testGrain, sGrain, "Should be same / intern'ed GrainId object");
ActivationId testActivation = ActivationId.GetSystemActivation(testGrain, SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 2456), 0));
Console.WriteLine("Testing ActivationID " + testActivation);
Assert.IsTrue(testActivation.IsSystem, "System activation ID is not flagged as a system ID");
}
/// <summary>
/// Finds dead silos which were previously in an unknown state.
/// </summary>
/// <param name="allKnownSilos">The collection of all known silos, including dead silos.</param>
/// <returns>A collection of dead silos.</returns>
public IEnumerable <SiloAddress> DetermineDeadSilos(Dictionary <SiloAddress, SiloStatus> allKnownSilos)
{
if (this.unknownSilos.Count == 0)
{
return(Array.Empty <SiloAddress>());
}
// The latest generation for each silo endpoint.
var latestGenerations = new Dictionary <IPEndPoint, int>();
// All known silos can be removed from the unknown list as long as their status is valid.
foreach (var known in allKnownSilos)
{
if (known.Value == SiloStatus.None)
{
continue;
}
var address = known.Key;
var endpoint = address.Endpoint;
if (!latestGenerations.TryGetValue(endpoint, out var knownGeneration) || knownGeneration < address.Generation)
{
latestGenerations[endpoint] = address.Generation;
}
// Unknown silos are not removed from the collection until they have been confirmed in the collection of known silos.
if (this.unknownSilos.TryRemove(address, out var _))
{
this.log.Info($"Previously unknown silo {address} has transitioned to state {known.Value}.");
}
}
var updates = new List <SiloAddress>();
foreach (var pair in this.unknownSilos)
{
var unknownSilo = pair.Key;
// If a known silo exists on the endpoint with a higher generation, the old silo must be dead.
if (latestGenerations.TryGetValue(unknownSilo.Endpoint, out var knownGeneration) && knownGeneration > unknownSilo.Generation)
{
this.log.Info($"Previously unknown silo {unknownSilo} was superseded by later generation on same endpoint {SiloAddress.New(unknownSilo.Endpoint, knownGeneration)}.");
updates.Add(unknownSilo);
}
// Silos which have been in an unknown state for more than configured maximum allowed time are automatically considered dead.
if (this.GetDateTime() - pair.Value > this.options.UnknownSiloRemovalPeriod)
{
this.log.Info($"Previously unknown silo {unknownSilo} declared dead after {this.options.UnknownSiloRemovalPeriod.TotalSeconds} seconds.");
updates.Add(unknownSilo);
}
}
if (this.unknownSilos.Count == 0 && updates.Count == 0)
{
this.log.Info("All unknown silos have been identified.");
}
return(updates);
}
private static SystemTargetGrainId CreateId(ILocalSiloDetails localSiloDetails)
{
return(SystemTargetGrainId.Create(Constants.SystemMembershipTableType, SiloAddress.New(localSiloDetails.SiloAddress.Endpoint, 0)));
}
public async Task HandlesSiloAdditionAndRemoval()
{
var listener = new MockStatusListener();
this.oracle.SubscribeToSiloStatusEvents(listener);
await this.oracle.Start();
await this.oracle.BecomeActive();
var multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(1, multiClusters.Count);
Assert.Contains(this.siloDetails.SiloAddress, multiClusters);
var silos = new[]
{
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 1), 1),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 2), 1),
"HappyNewSilo"),
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 3), 2),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 4), 2),
"OtherNewSilo"),
};
// The local silo transitioned to 'Joining' and then 'Active' and two other silos became 'Active',
// so the version must be 4
this.resolver.Notify(silos);
listener.WaitForVersion(4);
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Active);
AssertStatus(silos[1].SiloAddress, SiloStatus.Active);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Send the same update again and verify that nothing changed.
this.resolver.Notify(silos);
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Remove a silo and verify that it's been removed.
// The single removal will bump the version to 5.
this.resolver.Notify(new[] { silos[1] });
listener.WaitForVersion(5);
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[1].SiloAddress, SiloStatus.Active);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[0].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Dead);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Remove a silo and verify that it's been removed.
this.resolver.Notify(new FabricSiloInfo[0]);
listener.WaitForVersion(6);
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[0].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Dead);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[1].SiloAddress, SiloStatus.Dead);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(1, multiClusters.Count);
Assert.Contains(this.siloDetails.SiloAddress, multiClusters);
}
private void RunTest(int numItems)
{
InvokeMethodRequest request = new InvokeMethodRequest(0, 0, null);
Message resp = Message.CreateMessage(request, InvokeMethodOptions.None);
resp.Id = new CorrelationId();
resp.SendingSilo = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 200), 0);
resp.TargetSilo = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 300), 0);
resp.SendingGrain = GrainId.NewId();
resp.TargetGrain = GrainId.NewId();
resp.IsAlwaysInterleave = true;
List <object> requestBody = new List <object>();
for (int k = 0; k < numItems; k++)
{
requestBody.Add(k + ": test line");
}
resp.BodyObject = requestBody;
string s = resp.ToString();
output.WriteLine(s);
int dummy = 0;
var serialized = resp.Serialize(out dummy);
int length = serialized.Sum <ArraySegment <byte> >(x => x.Count);
byte[] data = new byte[length];
int n = 0;
foreach (var buffer in serialized)
{
Array.Copy(buffer.Array, buffer.Offset, data, n, buffer.Count);
n += buffer.Count;
}
resp.ReleaseBodyAndHeaderBuffers();
int headerLength = BitConverter.ToInt32(data, 0);
int bodyLength = BitConverter.ToInt32(data, 4);
Assert.Equal <int>(length, headerLength + bodyLength + 8); //Serialized lengths are incorrect
byte[] header = new byte[headerLength];
Array.Copy(data, 8, header, 0, headerLength);
byte[] body = new byte[bodyLength];
Array.Copy(data, 8 + headerLength, body, 0, bodyLength);
var headerList = new List <ArraySegment <byte> >();
headerList.Add(new ArraySegment <byte>(header));
var bodyList = new List <ArraySegment <byte> >();
bodyList.Add(new ArraySegment <byte>(body));
var resp1 = new Message(headerList, bodyList);
//byte[] serialized = resp.FormatForSending();
//Message resp1 = new Message(serialized, serialized.Length);
Assert.Equal(resp.Category, resp1.Category); //Category is incorrect"
Assert.Equal(resp.Direction, resp1.Direction); //Direction is incorrect
Assert.Equal(resp.Id, resp1.Id); //Correlation ID is incorrect
Assert.Equal(resp.IsAlwaysInterleave, resp1.IsAlwaysInterleave); //Foo Boolean is incorrect
Assert.Equal(resp.CacheInvalidationHeader, resp1.CacheInvalidationHeader); //Bar string is incorrect
Assert.True(resp.TargetSilo.Equals(resp1.TargetSilo));
Assert.True(resp.TargetGrain.Equals(resp1.TargetGrain));
Assert.True(resp.SendingGrain.Equals(resp1.SendingGrain));
Assert.True(resp.SendingSilo.Equals(resp1.SendingSilo)); //SendingSilo is incorrect
List <object> responseList = Assert.IsAssignableFrom <List <object> >(resp1.BodyObject);
Assert.Equal <int>(numItems, responseList.Count); //Body list has wrong number of entries
for (int k = 0; k < numItems; k++)
{
Assert.IsAssignableFrom <string>(responseList[k]); //Body list item " + k + " has wrong type
Assert.Equal <string>((string)(requestBody[k]), (string)(responseList[k])); //Body list item " + k + " is incorrect
}
}
public void MultiClusterData_Gateways()
{
var ts1 = DateTime.UtcNow;
var ts2 = ts1 + new TimeSpan(hours: 0, minutes: 0, seconds: 1);
var ts3 = ts1 + new TimeSpan(hours: 0, minutes: 0, seconds: 2);
IPAddress ip;
Assert.True(IPAddress.TryParse("127.0.0.1", out ip));
IPEndPoint ep1 = new IPEndPoint(ip, 21111);
var siloAddress1 = SiloAddress.New(ep1, 0);
IPEndPoint ep2 = new IPEndPoint(ip, 21112);
var siloAddress2 = SiloAddress.New(ep2, 0);
var G1 = new GatewayEntry()
{
SiloAddress = siloAddress1,
ClusterId = "1",
HeartbeatTimestamp = ts1,
Status = GatewayStatus.Active
};
var G2 = new GatewayEntry()
{
SiloAddress = siloAddress1,
ClusterId = "1",
HeartbeatTimestamp = ts3,
Status = GatewayStatus.Inactive
};
var H1 = new GatewayEntry()
{
SiloAddress = siloAddress2,
ClusterId = "2",
HeartbeatTimestamp = ts2,
Status = GatewayStatus.Active
};
var H2 = new GatewayEntry()
{
SiloAddress = siloAddress2,
ClusterId = "2",
HeartbeatTimestamp = ts3,
Status = GatewayStatus.Inactive
};
var gd1 = new MultiClusterData();
var gd2 = new MultiClusterData(G1);
var gd3 = new MultiClusterData(G2);
TestAlgebraicProperties(gd1, gd1);
TestAlgebraicProperties(gd2, gd2);
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd3, gd1);
TestAlgebraicProperties(gd2, gd3);
TestAlgebraicProperties(gd3, gd2);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { H2, G1 });
TestAlgebraicProperties(gd1, gd1);
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { H1, G1 });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { G1 });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { H2 });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
}
// This function is called asynchronously from gateway refresh timer.
private async Task UpdateLiveGatewaysSnapshot(IEnumerable <SiloAddress> refreshedGateways, TimeSpan maxStaleness)
{
List <SiloAddress> connectionsToKeepAlive;
// This is a short lock, protecting the access to knownDead, knownMasked and cachedLiveGateways.
lock (lockable)
{
// now take whatever listProvider gave us and exclude those we think are dead.
var live = new List <SiloAddress>();
var now = DateTime.UtcNow;
this.knownGateways = refreshedGateways as List <SiloAddress> ?? refreshedGateways.ToList();
foreach (SiloAddress trial in knownGateways)
{
var address = trial.Generation == 0 ? trial : SiloAddress.New(trial.Endpoint, 0);
// We consider a node to be dead if we recorded it is dead due to socket error
// and it was recorded (diedAt) not too long ago (less than maxStaleness ago).
// The latter is to cover the case when the Gateway provider returns an outdated list that does not yet reflect the actually recently died Gateway.
// If it has passed more than maxStaleness - we assume maxStaleness is the upper bound on Gateway provider freshness.
var isDead = false;
if (knownDead.TryGetValue(address, out var diedAt))
{
if (now.Subtract(diedAt) < maxStaleness)
{
isDead = true;
}
else
{
// Remove stale entries.
knownDead.Remove(address);
}
}
if (knownMasked.TryGetValue(address, out var maskedAt))
{
if (now.Subtract(maskedAt) < maxStaleness)
{
isDead = true;
}
else
{
// Remove stale entries.
knownMasked.Remove(address);
}
}
if (!isDead)
{
live.Add(address);
}
}
if (live.Count == 0)
{
logger.LogWarning(
(int)ErrorCode.GatewayManager_AllGatewaysDead,
"All gateways have previously been marked as dead. Clearing the list of dead gateways to expedite reconnection.");
live.AddRange(knownGateways);
knownDead.Clear();
}
// swap cachedLiveGateways pointer in one atomic operation
cachedLiveGateways = live;
cachedLiveGatewaysSet = new HashSet <SiloAddress>(live);
DateTime prevRefresh = lastRefreshTime;
lastRefreshTime = now;
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug(
(int)ErrorCode.GatewayManager_FoundKnownGateways,
"Refreshed the live gateway list. Found {KnownGatewayCount} gateways from gateway list provider: {KnownGateways}. Picked only known live out of them. Now has {LiveGatewayCount} live gateways: {LiveGateways}. Previous refresh time was = {PreviousRefreshTime}",
knownGateways.Count,
Utils.EnumerableToString(knownGateways),
cachedLiveGateways.Count,
Utils.EnumerableToString(cachedLiveGateways),
prevRefresh);
}
// Close connections to known dead connections, but keep the "masked" ones.
// Client will not send any new request to the "masked" connections, but might still
// receive responses
connectionsToKeepAlive = new List <SiloAddress>(live);
connectionsToKeepAlive.AddRange(knownMasked.Select(e => e.Key));
}
await this.CloseEvictedGatewayConnections(connectionsToKeepAlive);
}
public async Task HandlesSiloAdditionAndRemoval()
{
var listener = new MockStatusListener();
this.oracle.SubscribeToSiloStatusEvents(listener);
await this.oracle.Start();
await this.oracle.BecomeActive();
var multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(1, multiClusters.Count);
Assert.Contains(this.siloDetails.SiloAddress, multiClusters);
var silos = new[]
{
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 1), 1),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 2), 1),
"HappyNewSilo"),
CreateSiloInfo(
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 3), 2),
SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 4), 2),
"OtherNewSilo"),
};
listener.VersionReached.Reset();
listener.WaitForVersion = 4;
this.resolver.Notify(silos);
listener.VersionReached.WaitOne(TimeSpan.FromMinutes(1));
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Active);
AssertStatus(silos[1].SiloAddress, SiloStatus.Active);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Send the same update again and verify that nothing changed.
this.resolver.Notify(silos);
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Remove a silo and verify that it's been removed.
listener.WaitForVersion = 5;
this.resolver.Notify(new[] { silos[1] });
listener.VersionReached.WaitOne(TimeSpan.FromMinutes(1));
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Active, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[1].SiloAddress, SiloStatus.Active);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[0].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Dead);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(2, multiClusters.Count);
// Remove a silo and verify that it's been removed.
this.resolver.Notify(new FabricSiloInfo[0]);
listener.VersionReached.WaitOne(TimeSpan.FromMinutes(1));
Assert.Equal(3, listener.Silos.Count);
Assert.Contains(silos[1].SiloAddress, listener.Silos.Keys);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[0].SiloAddress]);
AssertStatus(silos[0].SiloAddress, SiloStatus.Dead);
Assert.Equal(SiloStatus.Dead, listener.Silos[silos[1].SiloAddress]);
AssertStatus(silos[1].SiloAddress, SiloStatus.Dead);
multiClusters = this.oracle.GetApproximateMultiClusterGateways();
Assert.Equal(1, multiClusters.Count);
Assert.Contains(this.siloDetails.SiloAddress, multiClusters);
}
