private AzureTableBasedGossipChannel gossipTable; // This type is internal
public AzureGossipTableTests()
{
logger = LogManager.GetLogger("AzureGossipTableTests", LoggerType.Application);
globalServiceId = Guid.NewGuid();
deploymentId = "test-" + globalServiceId;
IPAddress ip;
if (!IPAddress.TryParse("127.0.0.1", out ip))
{
logger.Error(-1, "Could not parse ip address");
return;
}
IPEndPoint ep1 = new IPEndPoint(ip, 21111);
siloAddress1 = SiloAddress.New(ep1, 0);
IPEndPoint ep2 = new IPEndPoint(ip, 21112);
siloAddress2 = SiloAddress.New(ep2, 0);
logger.Info("DeploymentId={0}", deploymentId);
GlobalConfiguration config = new GlobalConfiguration
{
ServiceId = globalServiceId,
ClusterId = "0",
DeploymentId = deploymentId,
DataConnectionString = TestDefaultConfiguration.DataConnectionString
};
gossipTable = new AzureTableBasedGossipChannel();
var done = gossipTable.Initialize(config.ServiceId, config.DataConnectionString);
if (!done.Wait(timeout))
{
throw new TimeoutException("Could not create/read table.");
}
}
internal TypeManager(
SiloAddress myAddr,
GrainTypeManager grainTypeManager,
ISiloStatusOracle oracle,
OrleansTaskScheduler scheduler,
TimeSpan refreshClusterMapTimeout,
ImplicitStreamSubscriberTable implicitStreamSubscriberTable)
: base(Constants.TypeManagerId, myAddr)
{
if (grainTypeManager == null)
throw new ArgumentNullException(nameof(grainTypeManager));
if (oracle == null)
throw new ArgumentNullException(nameof(oracle));
if (scheduler == null)
throw new ArgumentNullException(nameof(scheduler));
if (implicitStreamSubscriberTable == null)
throw new ArgumentNullException(nameof(implicitStreamSubscriberTable));
this.grainTypeManager = grainTypeManager;
this.statusOracle = oracle;
this.implicitStreamSubscriberTable = implicitStreamSubscriberTable;
this.scheduler = scheduler;
this.hasToRefreshClusterGrainInterfaceMap = true;
this.refreshClusterGrainInterfaceMapTimer = new AsyncTaskSafeTimer(
OnRefreshClusterMapTimer,
null,
TimeSpan.Zero, // Force to do it once right now
refreshClusterMapTimeout);
}
internal VirtualBucketsRingProvider(SiloAddress siloAddr, int nBucketsPerSilo)
{
if (nBucketsPerSilo <= 0 )
throw new IndexOutOfRangeException("numBucketsPerSilo is out of the range. numBucketsPerSilo = " + nBucketsPerSilo);
logger = TraceLogger.GetLogger(typeof(VirtualBucketsRingProvider).Name);
statusListeners = new List<IRingRangeListener>();
bucketsMap = new SortedDictionary<uint, SiloAddress>();
sortedBucketsList = new List<Tuple<uint, SiloAddress>>();
myAddress = siloAddr;
numBucketsPerSilo = nBucketsPerSilo;
lockable = new object();
running = true;
myRange = RangeFactory.CreateFullRange();
logger.Info("Starting {0} on silo {1}.", typeof(VirtualBucketsRingProvider).Name, siloAddr.ToStringWithHashCode());
StringValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_RING, ToString);
IntValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_RINGSIZE, () => GetRingSize());
StringValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_MYRANGE_RINGDISTANCE, () => String.Format("x{0,8:X8}", ((IRingRangeInternal)myRange).RangeSize()));
FloatValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_MYRANGE_RINGPERCENTAGE, () => (float)((IRingRangeInternal)myRange).RangePercentage());
FloatValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_AVERAGERINGPERCENTAGE, () =>
{
int size = GetRingSize();
return size == 0 ? 0 : ((float)100.0/(float) size);
});
// add myself to the list of members
AddServer(myAddress);
}
private ActivationAddress(SiloAddress silo, GrainId grain, ActivationId activation, MultiClusterStatus status)
{
Silo = silo;
Grain = grain;
Activation = activation;
Status = status;
}
public static ActivationAddress GetAddress(SiloAddress silo, GrainId grain, ActivationId activation, MultiClusterStatus status = MultiClusterStatus.Owned)
{
// Silo part is not mandatory
if (grain == null) throw new ArgumentNullException("grain");
return new ActivationAddress(silo, grain, activation, status);
}
protected SystemTarget(GrainId grainId, SiloAddress silo, bool lowPriority)
{
GrainId = grainId;
Silo = silo;
ActivationId = ActivationId.GetSystemActivation(grainId, silo);
SchedulingContext = new SchedulingContext(this, lowPriority);
}
public void TestMultiClusterGatewaySelection()
{
var candidates = new SiloAddress[] {
SiloAddress.New(new IPEndPoint(0,0),0),
SiloAddress.New(new IPEndPoint(0,0),1),
SiloAddress.New(new IPEndPoint(0,1),0),
SiloAddress.New(new IPEndPoint(0,1),1),
SiloAddress.New(new IPEndPoint(0,234),1),
SiloAddress.New(new IPEndPoint(1,0),0),
SiloAddress.New(new IPEndPoint(1,0),1),
SiloAddress.New(new IPEndPoint(1,1),1),
SiloAddress.New(new IPEndPoint(1,234),1),
SiloAddress.New(new IPEndPoint(2,234),1),
SiloAddress.New(new IPEndPoint(3,234),1),
SiloAddress.New(new IPEndPoint(4,234),1),
};
Func<SiloAddress,UpdateFaultCombo> group = (SiloAddress a) => new UpdateFaultCombo(a.Endpoint.Port, a.Generation);
// randomize order
var r = new Random();
var randomized = new SortedList<int,SiloAddress> ();
foreach(var c in candidates)
randomized.Add(r.Next(), c);
var x = MembershipOracleData.DeterministicBalancedChoice(randomized.Values, 10, group);
for (int i = 0; i < 10; i++)
Assert.Equal(candidates[i], x[i]);
}
public static ActivationAddress GetAddress(SiloAddress silo, GrainId grain, ActivationId activation)
{
// Silo part is not mandatory
if (grain == null) throw new ArgumentNullException("grain");
return new ActivationAddress(silo, grain, activation);
}
public Task ForceGarbageCollection(SiloAddress[] siloAddresses)
{
var silos = GetSiloAddresses(siloAddresses);
logger.Info("Forcing garbage collection on {0}", Utils.EnumerableToString(silos));
List<Task> actionPromises = PerformPerSiloAction(silos,
s => GetSiloControlReference(s).ForceGarbageCollection());
return Task.WhenAll(actionPromises);
}
internal bool TryGetSiloName(SiloAddress siloAddress, out string siloName)
{
if (siloAddress.Equals(MyAddress))
{
siloName = SiloName;
return true;
}
return localNamesTableCopy.TryGetValue(siloAddress, out siloName);
}
internal ClientObserverRegistrar(SiloAddress myAddr, ILocalGrainDirectory dir, OrleansTaskScheduler scheduler, ClusterConfiguration config)
: base(Constants.ClientObserverRegistrarId, myAddr)
{
grainDirectory = dir;
myAddress = myAddr;
this.scheduler = scheduler;
orleansConfig = config;
logger = LogManager.GetLogger(typeof(ClientObserverRegistrar).Name);
}
public Task<Streams.ImplicitStreamSubscriberTable> GetImplicitStreamSubscriberTable(SiloAddress silo)
{
Streams.ImplicitStreamSubscriberTable table = SiloProviderRuntime.Instance.ImplicitStreamSubscriberTable;
if (null == table)
{
throw new InvalidOperationException("the implicit stream subscriber table is not initialized");
}
return Task.FromResult(table);
}
// Implementation of exception serialization with custom properties according to:
// http://stackoverflow.com/questions/94488/what-is-the-correct-way-to-make-a-custom-net-exception-serializable
protected DuplicateActivationException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
if (info != null)
{
ActivationToUse = (ActivationAddress) info.GetValue("ActivationToUse", typeof (ActivationAddress));
PrimaryDirectoryForGrain = (SiloAddress) info.GetValue("PrimaryDirectoryForGrain", typeof (SiloAddress));
}
}
public Task SetLogLevel(SiloAddress[] siloAddresses, string logName, int traceLevel)
{
var silos = GetSiloAddresses(siloAddresses);
logger.Info("SetLogLevel[{1}]={2} {0}", Utils.EnumerableToString(silos), logName, traceLevel);
List<Task> actionPromises = PerformPerSiloAction(silos,
s => GetSiloControlReference(s).SetLogLevel(logName, traceLevel));
return Task.WhenAll(actionPromises);
}
/// <summary>
/// Called when the status of a silo in the cluster changes.
/// - Notify listeners
/// </summary>
/// <param name="updatedSilo">Silo which status has changed</param>
/// <param name="status">new silo status</param>
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (status.Equals(SiloStatus.Dead))
{
// just clean up garbage from immatureSilos.
bool ignore;
immatureSilos.TryRemove(updatedSilo, out ignore);
}
SiloStatusChangeNotification().Ignore();
}
public async Task Init(string deploymentId, string storageConnectionString, SiloAddress siloAddress, string siloName, IPEndPoint gateway, string hostName)
{
this.deploymentId = deploymentId;
this.siloAddress = siloAddress;
this.siloName = siloName;
this.gateway = gateway;
myHostName = hostName;
storage = new AzureTableDataManager<SiloMetricsData>( INSTANCE_TABLE_NAME, storageConnectionString, logger);
await storage.InitTableAsync().WithTimeout(initTimeout);
}
public void AddConfiguration(string deployment, bool silo, string siloId, SiloAddress address, IPEndPoint gatewayAddress, string hostName)
{
deploymentId = deployment;
isSilo = silo;
siloName = siloId;
siloAddress = address;
gateway = gatewayAddress;
myHostName = hostName;
if (!isSilo)
generation = SiloAddress.AllocateNewGeneration();
}
public void AddConfiguration(string deploymentId, bool isSilo, string siloName, SiloAddress address, IPEndPoint gateway, string hostName)
{
SiloName = siloName;
DeploymentId = deploymentId;
HostName = hostName;
TelemetryConfiguration.Active.ContextInitializers.Add(new AppInInitializer(DeploymentId));
var tc = new TelemetryConfiguration();
Telemetry = new TelemetryClient();
Telemetry.InstrumentationKey = InstrumentationKey;
Initialized = true;
}
public void TestInitialize()
{
deploymentId = "test-" + Guid.NewGuid();
generation = SiloAddress.AllocateNewGeneration();
siloAddress = SiloAddress.NewLocalAddress(generation);
logger.Info("DeploymentId={0} Generation={1}", deploymentId, generation);
logger.Info("Initializing SiloInstanceManager");
manager = OrleansSiloInstanceManager.GetManager(deploymentId, StorageTestConstants.DataConnectionString)
.WaitForResultWithThrow(SiloInstanceTableTestConstants.Timeout);
}
Task<MembershipTableData> IMembershipTable.ReadRow(SiloAddress key)
{
if (logger.IsVerbose3) logger.Verbose3(string.Format("SqlMembershipTable.ReadRow called with key: {0}.", key));
try
{
return orleansQueries.MembershipDataAsync(deploymentId, key);
}
catch(Exception ex)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.ReadRow failed: {0}", ex);
throw;
}
}
public ClientObserverRegistrar(
SiloInitializationParameters initializationParameters,
ILocalGrainDirectory dir,
OrleansTaskScheduler scheduler,
ClusterConfiguration config)
: base(Constants.ClientObserverRegistrarId, initializationParameters.SiloAddress)
{
grainDirectory = dir;
myAddress = initializationParameters.SiloAddress;
this.scheduler = scheduler;
orleansConfig = config;
logger = LogManager.GetLogger(typeof(ClientObserverRegistrar).Name);
}
public ConsistentRingProvider(SiloAddress siloAddr)
{
log = LogManager.GetLogger(typeof(ConsistentRingProvider).Name);
membershipRingList = new List<SiloAddress>();
MyAddress = siloAddr;
myKey = MyAddress.GetConsistentHashCode();
// add myself to the list of members
AddServer(MyAddress);
MyRange = RangeFactory.CreateFullRange(); // i am responsible for the whole range
statusListeners = new List<IRingRangeListener>();
Start();
}
public ActivationAddress AddSingleActivation(GrainId grain, ActivationId act, SiloAddress silo)
{
SingleInstance = true;
if (Instances.Count > 0)
{
var item = Instances.First();
return ActivationAddress.GetAddress(item.Value.SiloAddress, grain, item.Key);
}
else
{
Instances.Add(act, new ActivationInfo(silo));
VersionTag = rand.Next();
return ActivationAddress.GetAddress(silo, grain, act);
}
}
public Task UpdateRuntimeStatistics(SiloAddress siloAddress, SiloRuntimeStatistics siloStats)
{
if (logger.IsVerbose) logger.Verbose("UpdateRuntimeStatistics from {0}", siloAddress);
if (silo.LocalSiloStatusOracle.GetApproximateSiloStatus(siloAddress) != SiloStatus.Active)
return TaskDone.Done;
SiloRuntimeStatistics old;
// Take only if newer.
if (periodicStats.TryGetValue(siloAddress, out old) && old.DateTime > siloStats.DateTime)
return TaskDone.Done;
periodicStats[siloAddress] = siloStats;
NotifyAllStatisticsChangeEventsSubscribers(siloAddress, siloStats);
return TaskDone.Done;
}
public ActivationAddress AddSingleActivation(GrainId grain, ActivationId act, SiloAddress silo, MultiClusterStatus registrationStatus = MultiClusterStatus.Owned)
{
SingleInstance = true;
if (Instances.Count > 0)
{
var item = Instances.First();
return ActivationAddress.GetAddress(item.Value.SiloAddress, grain, item.Key);
}
else
{
Instances.Add(act, new ActivationInfo(silo, registrationStatus));
VersionTag = rand.Next();
return ActivationAddress.GetAddress(silo, grain, act, registrationStatus);
}
}
public async Task<MembershipTableData> ReadRow(SiloAddress key)
{
try
{
var entries = await tableManager.FindSiloEntryAndTableVersionRow(key);
MembershipTableData data = Convert(entries);
if (logger.IsVerbose2) logger.Verbose2("Read my entry {0} Table=" + Environment.NewLine + "{1}", key.ToLongString(), data.ToString());
return data;
}
catch (Exception exc)
{
logger.Warn(ErrorCode.AzureTable_20, String.Format("Intermediate error reading silo entry for key {0} from the table {1}.",
key.ToLongString(), tableManager.TableName), exc);
throw;
}
}
internal static async Task MembershipTable_ReadRow_EmptyTable(IMembershipTable membership, SiloAddress siloAddress)
{
MembershipTableData data = await membership.ReadRow(siloAddress);
TableVersion tableVersion = data.Version;
logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", tableVersion, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
string eTag = tableVersion.VersionEtag;
int ver = tableVersion.Version;
logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} TableVersion={1}", eTag, tableVersion);
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.AreEqual(0, ver, "Initial table version should be zero");
}
public Task<List<Tuple<GrainId, int, List<Tuple<SiloAddress, ActivationId>>>>> LookUpMany(
SiloAddress destination, List<Tuple<GrainId, int>> grainAndETagList, int retries = 0)
{
var list = new List<Tuple<GrainId, int, List<Tuple<SiloAddress, ActivationId>>>>();
foreach (Tuple<GrainId, int> tuple in grainAndETagList)
{
GrainId id = tuple.Item1;
var reply = new List<Tuple<SiloAddress, ActivationId>>();
for (int i = 0; i < 10; i++)
{
reply.Add(new Tuple<SiloAddress, ActivationId>(SiloAddress.NewLocalAddress(0), ActivationId.NewId()));
}
list.Add(new Tuple<GrainId, int, List<Tuple<SiloAddress, ActivationId>>>(id, 3, reply));
}
return Task.FromResult(list);
}
private async Task Initialize()
{
deploymentId = "test-" + Guid.NewGuid();
int generation = SiloAddress.AllocateNewGeneration();
siloAddress = SiloAddress.NewLocalAddress(generation);
logger.Info("DeploymentId={0} Generation={1}", deploymentId, generation);
GlobalConfiguration config = new GlobalConfiguration
{
DeploymentId = deploymentId,
DataConnectionString = relationalStorage.ConnectionString
};
var mbr = new SqlMembershipTable();
await mbr.InitializeMembershipTable(config, true, logger).WithTimeout(timeout);
membership = mbr;
}
private async Task Initialize()
{
deploymentId = "test-" + Guid.NewGuid();
int generation = SiloAddress.AllocateNewGeneration();
siloAddress = SiloAddress.NewLocalAddress(generation);
logger.Info("DeploymentId={0} Generation={1}", deploymentId, generation);
GlobalConfiguration config = new GlobalConfiguration
{
DeploymentId = deploymentId,
DataConnectionString = StorageTestConstants.GetZooKeeperConnectionString()
};
var mbr = AssemblyLoader.LoadAndCreateInstance<IMembershipTable>(Constants.ORLEANS_ZOOKEEPER_UTILS_DLL, logger);
await mbr.InitializeMembershipTable(config, true, logger).WithTimeout(timeout);
membership = mbr;
}
private async Task <List <SiloHandle> > getSilosToFail(Fail fail, int numOfFailures)
{
List <SiloHandle> failures = new List <SiloHandle>();
int count = 0, index = 0;
// Figure out the primary directory partition and the silo hosting the ReminderTableGrain.
var tableGrain = this.GrainFactory.GetGrain <IReminderTableGrain>(Constants.ReminderTableGrainId);
var tableGrainId = ((GrainReference)tableGrain).GrainId;
SiloAddress reminderTableGrainPrimaryDirectoryAddress = (await TestUtils.GetDetailedGrainReport(this.HostedCluster.InternalGrainFactory, tableGrainId, this.HostedCluster.Primary)).PrimaryForGrain;
// ask a detailed report from the directory partition owner, and get the actionvation addresses
var addresses = (await TestUtils.GetDetailedGrainReport(this.HostedCluster.InternalGrainFactory, tableGrainId, this.HostedCluster.GetSiloForAddress(reminderTableGrainPrimaryDirectoryAddress))).LocalDirectoryActivationAddresses;
ActivationAddress reminderGrainActivation = addresses.FirstOrDefault();
SortedList <int, SiloHandle> ids = new SortedList <int, SiloHandle>();
foreach (var siloHandle in this.HostedCluster.GetActiveSilos())
{
SiloAddress siloAddress = siloHandle.SiloAddress;
if (siloAddress.Equals(this.HostedCluster.Primary.SiloAddress))
{
continue;
}
// Don't fail primary directory partition and the silo hosting the ReminderTableGrain.
if (siloAddress.Equals(reminderTableGrainPrimaryDirectoryAddress) || siloAddress.Equals(reminderGrainActivation.Silo))
{
continue;
}
ids.Add(siloHandle.SiloAddress.GetConsistentHashCode(), siloHandle);
}
// we should not fail the primary!
// we can't guarantee semantics of 'Fail' if it evalutes to the primary's address
switch (fail)
{
case Fail.First:
index = 0;
while (count++ < numOfFailures)
{
while (failures.Contains(ids.Values[index]))
{
index++;
}
failures.Add(ids.Values[index]);
}
break;
case Fail.Last:
index = ids.Count - 1;
while (count++ < numOfFailures)
{
while (failures.Contains(ids.Values[index]))
{
index--;
}
failures.Add(ids.Values[index]);
}
break;
case Fail.Random:
default:
while (count++ < numOfFailures)
{
SiloHandle r = ids.Values[random.Next(ids.Count)];
while (failures.Contains(r))
{
r = ids.Values[random.Next(ids.Count)];
}
failures.Add(r);
}
break;
}
return(failures);
}
public async Task <bool> DirectApplyIndexUpdate(IIndexableGrain g, Immutable <IMemberUpdate> iUpdate, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
IMemberUpdate update = iUpdate.Value;
IndexOperationType opType = update.GetOperationType();
if (opType == IndexOperationType.Update)
{
int befImgHash = update.GetBeforeImage().GetHashCode();
int aftImgHash = update.GetAfterImage().GetHashCode();
BucketT befImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + befImgHash
);
if (befImgHash == aftImgHash)
{
return(await befImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
else
{
BucketT aftImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + befImgHash
);
var befTask = befImgBucket.DirectApplyIndexUpdate(g, new MemberUpdateOverridenOperation(iUpdate.Value, IndexOperationType.Delete).AsImmutable <IMemberUpdate>(), isUniqueIndex, idxMetaData);
var aftTask = aftImgBucket.DirectApplyIndexUpdate(g, new MemberUpdateOverridenOperation(iUpdate.Value, IndexOperationType.Insert).AsImmutable <IMemberUpdate>(), isUniqueIndex, idxMetaData);
bool[] results = await Task.WhenAll(befTask, aftTask);
return(results[0] && results[1]);
}
}
else if (opType == IndexOperationType.Insert)
{
int aftImgHash = update.GetAfterImage().GetHashCode();
BucketT aftImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + aftImgHash
);
return(await aftImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
else if (opType == IndexOperationType.Delete)
{
int befImgHash = update.GetBeforeImage().GetHashCode();
BucketT befImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + befImgHash
);
return(await befImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
return(true);
}
public void RemoveSilo(SiloAddress removedSilo)
{
CachedLocalStat ignore;
localCache.TryRemove(removedSilo, out ignore);
}
internal static async System.Threading.Tasks.ValueTask Write(ConnectionContext connection, GrainId grainId, NetworkProtocolVersion protocolVersion, SiloAddress siloAddress)
{
var output = connection.Transport.Output;
var outputWriter = new PrefixingBufferWriter <byte, PipeWriter>(sizeof(int), 1024, MemoryPool <byte> .Shared);
outputWriter.Reset(output);
var writer = new BinaryTokenStreamWriter2 <PrefixingBufferWriter <byte, PipeWriter> >(outputWriter);
writer.Write(grainId);
writer.Write((byte)protocolVersion);
if (siloAddress is null)
{
writer.WriteNull();
}
else
{
writer.Write((byte)SerializationTokenType.SiloAddress);
writer.Write(siloAddress);
}
writer.Commit();
var length = outputWriter.CommittedBytes;
if (length > MaxPreambleLength)
{
throw new InvalidOperationException($"Created preamble of length {length}, which is greater than maximum allowed size of {MaxPreambleLength}.");
}
WriteLength(outputWriter, length);
var flushResult = await output.FlushAsync();
if (flushResult.IsCanceled)
{
throw new OperationCanceledException("Flush canceled");
}
return;
}
void IConfigurableStatisticsPublisher.AddConfiguration(string deploymentId, bool isSilo, string siloName, SiloAddress address, IPEndPoint gateway, string hostName)
{
}
public Task Init(string deploymentId, string storageConnectionString, SiloAddress siloAddress, string siloName, IPEndPoint gateway, string hostName)
{
return(TaskDone.Done);
}
public Task UpdateIAmAlive(string deploymentId, SiloAddress address, DateTime iAmAliveTime)
{
var id = ReturnId(deploymentId, address);
return(Collection.UpdateOneAsync(x => x.Id == id, Update.Set(x => x.IAmAliveTime, LogFormatter.PrintDate(iAmAliveTime))));
}
public Task RemoveHandoffPartition(SiloAddress source)
{
localGrainDirectory.HandoffManager.RemoveHandoffPartition(source);
return(Task.CompletedTask);
}
public ActivationAddress AddSingleActivation(GrainId grain, ActivationId act, SiloAddress silo)
{
SingleInstance = true;
if (Instances.Count > 0)
{
var item = Instances.First();
return(ActivationAddress.GetAddress(item.Value.SiloAddress, grain, item.Key));
}
else
{
Instances.Add(act, new ActivationInfo(silo));
VersionTag = rand.Next();
return(ActivationAddress.GetAddress(silo, grain, act));
}
}
/// <summary>
/// Adds a new activation to the directory partition
/// </summary>
/// <param name="grain"></param>
/// <param name="activation"></param>
/// <param name="silo"></param>
/// <returns>The registered ActivationAddress and version associated with this directory mapping</returns>
internal virtual Tuple <ActivationAddress, int> AddSingleActivation(GrainId grain, ActivationId activation, SiloAddress silo)
{
if (log.IsVerbose3)
{
log.Verbose3("Adding single activation for grain {0}{1}{2}", silo, grain, activation);
}
if (!IsValidSilo(silo))
{
return(null);
}
ActivationAddress result;
lock (lockable)
{
if (!partitionData.ContainsKey(grain))
{
partitionData[grain] = new GrainInfo();
}
result = partitionData[grain].AddSingleActivation(grain, activation, silo);
}
return(Tuple.Create(result, partitionData[grain].VersionTag));
}
public ActivationInfo(SiloAddress siloAddress)
{
SiloAddress = siloAddress;
TimeCreated = DateTime.UtcNow;
}
// This function is called asynchronously from gateway refresh timer.
private void UpdateLiveGatewaysSnapshot(IEnumerable <SiloAddress> refreshedGateways, TimeSpan maxStaleness)
{
// this is a short lock, protecting the access to knownDead and cachedLiveGateways.
lock (lockable)
{
// now take whatever listProvider gave us and exclude those we think are dead.
var live = new List <SiloAddress>();
var now = DateTime.UtcNow;
this.knownGateways = refreshedGateways as List <SiloAddress> ?? refreshedGateways.ToList();
foreach (SiloAddress trial in knownGateways)
{
var address = trial.Generation == 0 ? trial : SiloAddress.New(trial.Endpoint, 0);
// We consider a node to be dead if we recorded it is dead due to socket error
// and it was recorded (diedAt) not too long ago (less than maxStaleness ago).
// The latter is to cover the case when the Gateway provider returns an outdated list that does not yet reflect the actually recently died Gateway.
// If it has passed more than maxStaleness - we assume maxStaleness is the upper bound on Gateway provider freshness.
var isDead = false;
if (knownDead.TryGetValue(address, out var diedAt))
{
if (now.Subtract(diedAt) < maxStaleness)
{
isDead = true;
}
else
{
// Remove stale entries.
knownDead.Remove(address);
}
}
if (!isDead)
{
live.Add(address);
}
}
if (live.Count == 0)
{
logger.Warn(
ErrorCode.GatewayManager_AllGatewaysDead,
"All gateways have previously been marked as dead. Clearing the list of dead gateways to expedite reconnection.");
live.AddRange(knownGateways);
knownDead.Clear();
}
// swap cachedLiveGateways pointer in one atomic operation
cachedLiveGateways = live;
DateTime prevRefresh = lastRefreshTime;
lastRefreshTime = now;
if (logger.IsEnabled(LogLevel.Information))
{
logger.Info(ErrorCode.GatewayManager_FoundKnownGateways,
"Refreshed the live Gateway list. Found {0} gateways from Gateway listProvider: {1}. Picked only known live out of them. Now has {2} live Gateways: {3}. Previous refresh time was = {4}",
knownGateways.Count,
Utils.EnumerableToString(knownGateways),
cachedLiveGateways.Count,
Utils.EnumerableToString(cachedLiveGateways),
prevRefresh);
}
this.AbortEvictedGatewayConnections(live);
}
}
/// <summary>
/// Handles an incoming (proxied) message by rerouting it immediately and unconditionally,
/// after some header massaging.
/// </summary>
/// <param name="msg"></param>
/// <param name="receivedOnSocket"></param>
protected override void HandleMessage(Message msg, Socket receivedOnSocket)
{
// Don't process messages that have already timed out
if (msg.IsExpired)
{
msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Receive);
return;
}
if (Message.WriteMessagingTraces)
{
msg.AddTimestamp(Message.LifecycleTag.ReceiveIncoming);
}
gatewayTrafficCounter.Increment();
// Are we overloaded?
if ((MessageCenter.Metrics != null) && MessageCenter.Metrics.IsOverloaded)
{
MessagingStatisticsGroup.OnRejectedMessage(msg);
Message rejection = msg.CreateRejectionResponse(Message.RejectionTypes.GatewayTooBusy, "Shedding load");
MessageCenter.TryDeliverToProxy(rejection);
if (Log.IsVerbose)
{
Log.Verbose("Rejecting a request due to overloading: {0}", msg.ToString());
}
loadSheddingCounter.Increment();
return;
}
gateway.RecordSendingProxiedGrain(msg.SendingGrain, receivedOnSocket);
SiloAddress targetAddress = gateway.TryToReroute(msg);
msg.SendingSilo = MessageCenter.MyAddress;
if (targetAddress == null)
{
// reroute via Dispatcher
msg.RemoveHeader(Message.Header.TARGET_SILO);
msg.RemoveHeader(Message.Header.TARGET_ACTIVATION);
if (msg.TargetGrain.IsSystemTarget)
{
msg.TargetSilo = MessageCenter.MyAddress;
msg.TargetActivation = ActivationId.GetSystemActivation(msg.TargetGrain, MessageCenter.MyAddress);
}
if (Message.WriteMessagingTraces)
{
msg.AddTimestamp(Message.LifecycleTag.RerouteIncoming);
}
MessagingStatisticsGroup.OnMessageReRoute(msg);
MessageCenter.RerouteMessage(msg);
}
else
{
// send directly
msg.TargetSilo = targetAddress;
MessageCenter.SendMessage(msg);
}
}
public static string ConstructRowKey(SiloAddress silo, string clusterid)
{
return(String.Format(RowKeyFormat, clusterid, silo.Endpoint.Address, silo.Endpoint.Port, silo.Generation));
}
private static string ReturnId(string deploymentId, SiloAddress address)
{
return($"{deploymentId}@{ReturnAddress(address.Endpoint.Address)}:{address.Endpoint.Port}/{address.Generation}");
}
public Task AcceptHandoffPartition(SiloAddress source, Dictionary <GrainId, IGrainInfo> partition, bool isFullCopy)
{
localGrainDirectory.HandoffManager.AcceptHandoffPartition(source, partition, isFullCopy);
return(Task.CompletedTask);
}
// used for testing to (carefully!) allow two clients in the same process
private async Task StartInternal(Func <Exception, Task <bool> > retryFilter)
{
// Initialize the gateway list provider, since information from the cluster is required to successfully
// initialize subsequent services.
var initializedGatewayProvider = new[] { false };
await ExecuteWithRetries(async() =>
{
if (!initializedGatewayProvider[0])
{
await this.gatewayListProvider.InitializeGatewayListProvider();
initializedGatewayProvider[0] = true;
}
var gateways = await this.gatewayListProvider.GetGateways();
if (gateways.Count == 0)
{
var gatewayProviderType = this.gatewayListProvider.GetType().GetParseableName();
var err = $"Could not find any gateway in {gatewayProviderType}. Orleans client cannot initialize.";
logger.Error(ErrorCode.GatewayManager_NoGateways, err);
throw new SiloUnavailableException(err);
}
},
retryFilter);
var generation = -SiloAddress.AllocateNewGeneration(); // Client generations are negative
transport = ActivatorUtilities.CreateInstance <ClientMessageCenter>(this.ServiceProvider, localAddress, generation, handshakeClientId);
transport.RegisterLocalMessageHandler(Message.Categories.Application, this.HandleMessage);
transport.Start();
CurrentActivationAddress = ActivationAddress.NewActivationAddress(transport.MyAddress, handshakeClientId);
// Keeping this thread handling it very simple for now. Just queue task on thread pool.
Task.Run(this.RunClientMessagePump).Ignore();
await ExecuteWithRetries(
async() => this.GrainTypeResolver = await transport.GetGrainTypeResolver(this.InternalGrainFactory),
retryFilter);
this.typeMapRefreshTimer = new AsyncTaskSafeTimer(
this.logger,
RefreshGrainTypeResolver,
null,
this.typeMapRefreshInterval,
this.typeMapRefreshInterval);
ClientStatistics.Start(transport, clientId);
await ExecuteWithRetries(StreamingInitialize, retryFilter);
async Task ExecuteWithRetries(Func <Task> task, Func <Exception, Task <bool> > shouldRetry)
{
while (true)
{
try
{
await task();
return;
}
catch (Exception exception) when(shouldRetry != null)
{
var retry = await shouldRetry(exception);
if (!retry)
{
throw;
}
}
}
}
}
public static SiloAddress NewLocalSiloAddress(int gen)
{
return(SiloAddress.New(localEndpoint, gen));
}
public void Write(SiloAddress addr)
{
this.Write(addr.Endpoint);
this.Write(addr.Generation);
}
public async Task <MembershipTableData> ReadRow(SiloAddress siloAddress)
{
var siloRegistration = await GetConsulSiloRegistration(siloAddress);
return(AssembleMembershipTableData(siloRegistration));
}
public Task <MembershipTableData> ReadRow(SiloAddress key)
{
return(Task.FromResult(table.Read(key)));
}
private async Task <(ConsulSiloRegistration, TableVersion)> GetConsulSiloRegistration(SiloAddress siloAddress)
{
var deploymentKey = ConsulSiloRegistrationAssembler.FormatDeploymentKVPrefix(this.clusterId, this.kvRootFolder);
var siloKey = ConsulSiloRegistrationAssembler.FormatDeploymentSiloKey(this.clusterId, this.kvRootFolder, siloAddress);
var entries = await _consulClient.KV.List(deploymentKey);
if (entries.Response == null)
{
return(null, NotFoundTableVersion);
}
var siloKV = entries.Response.Single(KV => KV.Key.Equals(siloKey, StringComparison.OrdinalIgnoreCase));
var iAmAliveKV = entries.Response.SingleOrDefault(KV => KV.Key.Equals(ConsulSiloRegistrationAssembler.FormatSiloIAmAliveKey(siloKey), StringComparison.OrdinalIgnoreCase));
var tableVersion = GetTableVersion(versionKey: versionKey, entries: entries);
var siloRegistration = ConsulSiloRegistrationAssembler.FromKVPairs(this.clusterId, siloKV, iAmAliveKV);
return(siloRegistration, tableVersion);
}
internal CachedLocalStat(SiloAddress address, SiloRuntimeStatistics siloStats)
{
Address = address;
SiloStats = siloStats;
}
private ISiloControl GetSiloControlReference(SiloAddress silo)
{
return(this.internalGrainFactory.GetSystemTarget <ISiloControl>(Constants.SiloControlId, silo));
}
public async Task <bool> DirectApplyIndexUpdateBatch(Immutable <IDictionary <IIndexableGrain, IList <IMemberUpdate> > > iUpdates, bool isUnique, IndexMetaData idxMetaData, SiloAddress siloAddress = null)
{
if (logger.IsVerbose)
{
logger.Verbose("Started calling DirectApplyIndexUpdateBatch with the following parameters: isUnique = {0}, siloAddress = {1}, iUpdates = {2}", isUnique, siloAddress, MemberUpdate.UpdatesToString(iUpdates.Value));
}
IDictionary <IIndexableGrain, IList <IMemberUpdate> > updates = iUpdates.Value;
IDictionary <int, IDictionary <IIndexableGrain, IList <IMemberUpdate> > > bucketUpdates = new Dictionary <int, IDictionary <IIndexableGrain, IList <IMemberUpdate> > >();
foreach (var kv in updates)
{
IIndexableGrain g = kv.Key;
IList <IMemberUpdate> gUpdates = kv.Value;
foreach (IMemberUpdate update in gUpdates)
{
IndexOperationType opType = update.GetOperationType();
if (opType == IndexOperationType.Update)
{
int befImgHash = update.GetBeforeImage().GetHashCode();
int aftImgHash = update.GetAfterImage().GetHashCode();
if (befImgHash == aftImgHash)
{
AddUpdateToBucket(bucketUpdates, g, befImgHash, update);
}
else
{
AddUpdateToBucket(bucketUpdates, g, befImgHash, new MemberUpdateOverridenOperation(update, IndexOperationType.Delete));
AddUpdateToBucket(bucketUpdates, g, aftImgHash, new MemberUpdateOverridenOperation(update, IndexOperationType.Insert));
}
}
else if (opType == IndexOperationType.Insert)
{
int aftImgHash = update.GetAfterImage().GetHashCode();
AddUpdateToBucket(bucketUpdates, g, aftImgHash, update);
}
else if (opType == IndexOperationType.Delete)
{
int befImgHash = update.GetBeforeImage().GetHashCode();
AddUpdateToBucket(bucketUpdates, g, befImgHash, update);
}
}
}
List <Task> updateTasks = new List <Task>();
int i = 0;
foreach (var kv in bucketUpdates)
{
BucketT bucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + kv.Key
);
updateTasks.Add(bucket.DirectApplyIndexUpdateBatch(kv.Value.AsImmutable(), isUnique, idxMetaData, siloAddress));
++i;
}
await Task.WhenAll(updateTasks);
if (logger.IsVerbose)
{
logger.Verbose("Finished calling DirectApplyIndexUpdateBatch with the following parameters: isUnique = {0}, siloAddress = {1}, iUpdates = {2}", isUnique, siloAddress, MemberUpdate.UpdatesToString(iUpdates.Value));
}
return(true);
}
internal void RecordClientRoute(GrainId client, SiloAddress gateway)
{
var now = DateTime.UtcNow;
clientRoutes.AddOrUpdate(client, new Tuple <SiloAddress, DateTime>(gateway, now), (k, v) => new Tuple <SiloAddress, DateTime>(gateway, now));
}
public async Task <bool> TryToSuspectOrKill(SiloAddress silo)
{
var table = await membershipTableProvider.ReadAll();
if (log.IsEnabled(LogLevel.Debug))
{
log.Debug("-TryToSuspectOrKill: Read Membership table {0}", table.ToString());
}
if (this.IsStopping)
{
this.log.LogInformation(
(int)ErrorCode.MembershipFoundMyselfDead3,
"Ignoring call to TrySuspectOrKill for silo {Silo} since the local silo is dead",
silo);
return(true);
}
var(localSiloEntry, _) = this.GetOrCreateLocalSiloEntry(table, this.CurrentStatus);
if (localSiloEntry.Status == SiloStatus.Dead)
{
var msg = string.Format("I should be Dead according to membership table (in TryToSuspectOrKill): entry = {0}.", localSiloEntry.ToFullString(full: true));
log.Warn(ErrorCode.MembershipFoundMyselfDead3, msg);
KillMyselfLocally(msg);
return(true);
}
if (!table.Contains(silo))
{
// this should not happen ...
var str = string.Format("-Could not find silo entry for silo {0} in the table.", silo);
log.Error(ErrorCode.MembershipFailedToReadSilo, str);
throw new KeyNotFoundException(str);
}
var tuple = table.Get(silo);
var entry = tuple.Item1.Copy();
string eTag = tuple.Item2;
if (log.IsEnabled(LogLevel.Debug))
{
log.Debug("-TryToSuspectOrKill {siloAddress}: The current status of {siloAddress} in the table is {status}, its entry is {entry}",
entry.SiloAddress, // First
entry.SiloAddress, // Second
entry.Status,
entry.ToFullString());
}
// check if the table already knows that this silo is dead
if (entry.Status == SiloStatus.Dead)
{
this.ProcessTableUpdate(table, "TrySuspectOrKill");
return(true);
}
var allVotes = entry.SuspectTimes ?? new List <Tuple <SiloAddress, DateTime> >();
// get all valid (non-expired) votes
var freshVotes = entry.GetFreshVotes(DateTime.UtcNow, this.clusterMembershipOptions.DeathVoteExpirationTimeout);
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("-Current number of fresh Voters for {0} is {1}", silo, freshVotes.Count.ToString());
}
if (freshVotes.Count >= this.clusterMembershipOptions.NumVotesForDeathDeclaration)
{
// this should not happen ...
var str = string.Format("-Silo {0} is suspected by {1} which is more or equal than {2}, but is not marked as dead. This is a bug!!!",
entry.SiloAddress, freshVotes.Count.ToString(), this.clusterMembershipOptions.NumVotesForDeathDeclaration.ToString());
log.Error(ErrorCode.Runtime_Error_100053, str);
KillMyselfLocally("Found a bug! Will stop.");
return(false);
}
// handle the corner case when the number of active silos is very small (then my only vote is enough)
int activeSilos = table.GetSiloStatuses(status => status == SiloStatus.Active, true, this.localSiloDetails.SiloAddress).Count;
// find if I have already voted
int myVoteIndex = freshVotes.FindIndex(voter => myAddress.Equals(voter.Item1));
// Try to kill:
// if there is NumVotesForDeathDeclaration votes (including me) to kill - kill.
// otherwise, if there is a majority of nodes (including me) voting to kill – kill.
bool declareDead = false;
int myAdditionalVote = myVoteIndex == -1 ? 1 : 0;
if (freshVotes.Count + myAdditionalVote >= this.clusterMembershipOptions.NumVotesForDeathDeclaration)
{
declareDead = true;
}
if (freshVotes.Count + myAdditionalVote >= (activeSilos + 1) / 2)
{
declareDead = true;
}
if (declareDead)
{
// kick this silo off
log.Info(ErrorCode.MembershipMarkingAsDead,
"-Going to mark silo {0} as DEAD in the table #1. I am the last voter: #freshVotes={1}, myVoteIndex = {2}, NumVotesForDeathDeclaration={3} , #activeSilos={4}, suspect list={5}",
entry.SiloAddress,
freshVotes.Count,
myVoteIndex,
this.clusterMembershipOptions.NumVotesForDeathDeclaration,
activeSilos,
PrintSuspectList(allVotes));
return(await DeclareDead(entry, eTag, table.Version));
}
// we still do not have enough votes - need to vote
// find voting place:
// update my vote, if I voted previously
// OR if the list is not full - just add a new vote
// OR overwrite the oldest entry.
int indexToWrite = allVotes.FindIndex(voter => myAddress.Equals(voter.Item1));
if (indexToWrite == -1)
{
// My vote is not recorded. Find the most outdated vote if the list is full, and overwrite it
if (allVotes.Count >= this.clusterMembershipOptions.NumVotesForDeathDeclaration) // if the list is full
{
// The list is full.
DateTime minVoteTime = allVotes.Min(voter => voter.Item2); // pick the most outdated vote
indexToWrite = allVotes.FindIndex(voter => voter.Item2.Equals(minVoteTime));
}
}
var prevList = allVotes.ToList(); // take a copy
var now = DateTime.UtcNow;
if (indexToWrite == -1)
{
// if did not find specific place to write (the list is not full), just add a new element to the list
entry.AddSuspector(myAddress, now);
}
else
{
var newEntry = new Tuple <SiloAddress, DateTime>(myAddress, now);
entry.SuspectTimes[indexToWrite] = newEntry;
}
log.Info(ErrorCode.MembershipVotingForKill,
"-Putting my vote to mark silo {0} as DEAD #2. Previous suspect list is {1}, trying to update to {2}, eTag={3}, freshVotes is {4}",
entry.SiloAddress,
PrintSuspectList(prevList),
PrintSuspectList(entry.SuspectTimes),
eTag,
PrintSuspectList(freshVotes));
// If we fail to update here we will retry later.
return(await membershipTableProvider.UpdateRow(entry, eTag, table.Version.Next()));
string PrintSuspectList(IEnumerable <Tuple <SiloAddress, DateTime> > list)
{
return(Utils.EnumerableToString(list, t => string.Format("<{0}, {1}>",
t.Item1, LogFormatter.PrintDate(t.Item2))));
}
}
/// <summary>
/// Queries Orleans membership data.
/// </summary>
/// <param name="deploymentId">The deployment for which to query data.</param>
/// <param name="siloAddress">Silo data used as parameters in the query.</param>
/// <returns>Membership table data.</returns>
internal Task <MembershipTableData> MembershipReadRowAsync(string deploymentId, SiloAddress siloAddress)
{
return(ReadAsync(dbStoredQueries.MembershipReadRowKey, DbStoredQueries.Converters.GetMembershipEntry, command =>
new DbStoredQueries.Columns(command)
{
DeploymentId = deploymentId, SiloAddress = siloAddress
},
ConvertToMembershipTableData));
}
/// <inheritdoc />
TGrainInterface IInternalGrainFactory.GetSystemTarget <TGrainInterface>(GrainId grainId, SiloAddress destination)
{
return(this.InternalGrainFactory.GetSystemTarget <TGrainInterface>(grainId, destination));
}
/// <summary>
/// Find the silo handle for the specified silo address.
/// </summary>
/// <param name="siloAddress">Silo address to be found.</param>
/// <returns>SiloHandle of the appropriate silo, or <c>null</c> if not found.</returns>
public SiloHandle GetSiloForAddress(SiloAddress siloAddress)
{
var ret = GetActiveSilos().Where(s => s.Silo.SiloAddress.Equals(siloAddress)).FirstOrDefault();
return(ret);
}
