/// <summary>Returns a detailed human readable string that represents the current configuration. It does not contain every single configuration knob.</summary>
public override string ToString()
{
var sb = new StringBuilder();
sb.AppendLine("Platform version info:").Append(ConfigUtilities.RuntimeVersionInfo());
sb.Append(" Host: ").AppendLine(Dns.GetHostName());
sb.Append(" Processor Count: ").Append(System.Environment.ProcessorCount).AppendLine();
sb.AppendLine("Client Configuration:");
sb.Append(" Config File Name: ").AppendLine(string.IsNullOrEmpty(SourceFile) ? "" : Path.GetFullPath(SourceFile));
sb.Append(" Start time: ").AppendLine(LogFormatter.PrintDate(DateTime.UtcNow));
sb.Append(" Gateway Provider: ").Append(GatewayProvider);
if (GatewayProvider == GatewayProviderType.None)
{
sb.Append(". Gateway Provider that will be used instead: ").Append(GatewayProviderToUse);
}
sb.AppendLine();
if (Gateways != null && Gateways.Count > 0)
{
sb.AppendFormat(" Gateways[{0}]:", Gateways.Count).AppendLine();
foreach (var endpoint in Gateways)
{
sb.Append(" ").AppendLine(endpoint.ToString());
}
}
else
{
sb.Append(" Gateways: ").AppendLine("Unspecified");
}
sb.Append(" Preferred Gateway Index: ").AppendLine(PreferedGatewayIndex.ToString());
if (Gateways != null && PreferedGatewayIndex >= 0 && PreferedGatewayIndex < Gateways.Count)
{
sb.Append(" Preferred Gateway Address: ").AppendLine(Gateways[PreferedGatewayIndex].ToString());
}
sb.Append(" GatewayListRefreshPeriod: ").Append(GatewayListRefreshPeriod).AppendLine();
if (!String.IsNullOrEmpty(DeploymentId) || !String.IsNullOrEmpty(DataConnectionString))
{
sb.Append(" Azure:").AppendLine();
sb.Append(" DeploymentId: ").Append(DeploymentId).AppendLine();
string dataConnectionInfo = ConfigUtilities.RedactConnectionStringInfo(DataConnectionString); // Don't print Azure account keys in log files
sb.Append(" DataConnectionString: ").Append(dataConnectionInfo).AppendLine();
}
if (!string.IsNullOrWhiteSpace(NetInterface))
{
sb.Append(" Network Interface: ").AppendLine(NetInterface);
}
if (Port != 0)
{
sb.Append(" Network Port: ").Append(Port).AppendLine();
}
sb.Append(" Preferred Address Family: ").AppendLine(PreferredFamily.ToString());
sb.Append(" DNS Host Name: ").AppendLine(DNSHostName);
sb.Append(" Client Name: ").AppendLine(ClientName);
sb.Append(ConfigUtilities.TraceConfigurationToString(this));
sb.Append(ConfigUtilities.IStatisticsConfigurationToString(this));
sb.Append(LimitManager);
sb.AppendFormat(base.ToString());
#if !NETSTANDARD
sb.Append(" .NET: ").AppendLine();
int workerThreads;
int completionPortThreads;
ThreadPool.GetMinThreads(out workerThreads, out completionPortThreads);
sb.AppendFormat(" .NET thread pool sizes - Min: Worker Threads={0} Completion Port Threads={1}", workerThreads, completionPortThreads).AppendLine();
ThreadPool.GetMaxThreads(out workerThreads, out completionPortThreads);
sb.AppendFormat(" .NET thread pool sizes - Max: Worker Threads={0} Completion Port Threads={1}", workerThreads, completionPortThreads).AppendLine();
#endif
sb.AppendFormat(" Providers:").AppendLine();
sb.Append(ProviderConfigurationUtility.PrintProviderConfigurations(ProviderConfigurations));
return(sb.ToString());
}
/// <summary>
/// Initialize this Orleans silo for execution with the specified Azure clusterId
/// </summary>
/// <param name="config">If null, Config data will be read from silo config file as normal, otherwise use the specified config data.</param>
/// <param name="clusterId">Azure ClusterId this silo is running under</param>
/// <param name="connectionString">Azure DataConnectionString. If null, defaults to the DataConnectionString setting from the Azure configuration for this role.</param>
/// <returns><c>true</c> if the silo startup was successful</returns>
internal bool Start(ClusterConfiguration config, string clusterId, string connectionString)
{
if (config != null && clusterId != null)
{
throw new ArgumentException("Cannot use config and clusterId on the same time");
}
// Program ident
Trace.TraceInformation("Starting {0} v{1}", this.GetType().FullName, RuntimeVersion.Current);
// Read endpoint info for this instance from Azure config
string instanceName = serviceRuntimeWrapper.InstanceName;
// Configure this Orleans silo instance
if (config == null)
{
host = new SiloHost(instanceName);
host.LoadOrleansConfig(); // Load config from file + Initializes logger configurations
}
else
{
host = new SiloHost(instanceName, config); // Use supplied config data + Initializes logger configurations
}
IPEndPoint myEndpoint = serviceRuntimeWrapper.GetIPEndpoint(SiloEndpointConfigurationKeyName);
IPEndPoint proxyEndpoint = serviceRuntimeWrapper.GetIPEndpoint(ProxyEndpointConfigurationKeyName);
host.SetSiloType(Silo.SiloType.Secondary);
int generation = SiloAddress.AllocateNewGeneration();
// Bootstrap this Orleans silo instance
// If clusterId was not direclty provided, take the value in the config. If it is not
// in the config too, just take the ClusterId from Azure
if (clusterId == null)
{
clusterId = string.IsNullOrWhiteSpace(host.Config.Globals.ClusterId)
? serviceRuntimeWrapper.DeploymentId
: host.Config.Globals.ClusterId;
}
myEntry = new SiloInstanceTableEntry
{
DeploymentId = clusterId,
Address = myEndpoint.Address.ToString(),
Port = myEndpoint.Port.ToString(CultureInfo.InvariantCulture),
Generation = generation.ToString(CultureInfo.InvariantCulture),
HostName = host.Config.GetOrCreateNodeConfigurationForSilo(host.Name).DNSHostName,
ProxyPort = (proxyEndpoint != null ? proxyEndpoint.Port : 0).ToString(CultureInfo.InvariantCulture),
RoleName = serviceRuntimeWrapper.RoleName,
SiloName = instanceName,
UpdateZone = serviceRuntimeWrapper.UpdateDomain.ToString(CultureInfo.InvariantCulture),
FaultZone = serviceRuntimeWrapper.FaultDomain.ToString(CultureInfo.InvariantCulture),
StartTime = LogFormatter.PrintDate(DateTime.UtcNow),
PartitionKey = clusterId,
RowKey = myEndpoint.Address + "-" + myEndpoint.Port + "-" + generation
};
if (connectionString == null)
{
connectionString = serviceRuntimeWrapper.GetConfigurationSettingValue(DataConnectionConfigurationSettingName);
}
try
{
siloInstanceManager = OrleansSiloInstanceManager.GetManager(
clusterId, connectionString, AzureStorageClusteringOptions.DEFAULT_TABLE_NAME, this.loggerFactory).WithTimeout(AzureTableDefaultPolicies.TableCreationTimeout).Result;
}
catch (Exception exc)
{
var error = String.Format("Failed to create OrleansSiloInstanceManager. This means CreateTableIfNotExist for silo instance table has failed with {0}",
LogFormatter.PrintException(exc));
Trace.TraceError(error);
logger.Error((int)AzureSiloErrorCode.AzureTable_34, error, exc);
throw new OrleansException(error, exc);
}
// Always use Azure table for membership when running silo in Azure
host.SetSiloLivenessType(GlobalConfiguration.LivenessProviderType.AzureTable);
if (host.Config.Globals.ReminderServiceType == GlobalConfiguration.ReminderServiceProviderType.NotSpecified ||
host.Config.Globals.ReminderServiceType == GlobalConfiguration.ReminderServiceProviderType.ReminderTableGrain)
{
host.SetReminderServiceType(GlobalConfiguration.ReminderServiceProviderType.AzureTable);
}
host.SetExpectedClusterSize(serviceRuntimeWrapper.RoleInstanceCount);
siloInstanceManager.RegisterSiloInstance(myEntry);
// Initialize this Orleans silo instance
host.SetDeploymentId(clusterId, connectionString);
host.SetSiloEndpoint(myEndpoint, generation);
host.SetProxyEndpoint(proxyEndpoint);
host.ConfigureSiloHostDelegate = ConfigureSiloHostDelegate;
host.InitializeOrleansSilo();
return(StartSilo());
}
internal IStatement UpdateMembership(MembershipEntry membershipEntry, int version) =>
this["UpdateMembershipKey"].Bind(new
{
new_version = version + 1,
expected_version = version,
status = (int)membershipEntry.Status,
suspect_times = membershipEntry.SuspectTimes == null ? null : string.Join("|", membershipEntry.SuspectTimes.Select(s => $"{s.Item1.ToParsableString()},{LogFormatter.PrintDate(s.Item2)}")),
i_am_alive_time = membershipEntry.IAmAliveTime,
address = membershipEntry.SiloAddress.Endpoint.Address.ToString(),
port = membershipEntry.SiloAddress.Endpoint.Port,
generation = membershipEntry.SiloAddress.Generation
});
public async Task StatelessWorker()
{
IStatelessWorkerGrain grain = GrainClient.GrainFactory.GetGrain <IStatelessWorkerGrain>(0);
List <Task> promises = new List <Task>();
for (int i = 0; i < ExpectedMaxLocalActivations * 3; i++)
{
promises.Add(grain.LongCall()); // trigger activation of ExpectedMaxLocalActivations local activations
}
await Task.WhenAll(promises);
await Task.Delay(2000);
promises.Clear();
var stopwatch = Stopwatch.StartNew();
for (int i = 0; i < ExpectedMaxLocalActivations * 3; i++)
{
promises.Add(grain.LongCall());
}
await Task.WhenAll(promises);
stopwatch.Stop();
//Assert.IsTrue(stopwatch.Elapsed > TimeSpan.FromSeconds(19.5), "50 requests with a 2 second processing time shouldn't take less than 20 seconds on 10 activations. But it took " + stopwatch.Elapsed);
promises.Clear();
for (int i = 0; i < ExpectedMaxLocalActivations * 3; i++)
{
promises.Add(grain.GetCallStats()); // gather stats
}
await Task.WhenAll(promises);
HashSet <Guid> activations = new HashSet <Guid>();
foreach (var promise in promises)
{
Tuple <Guid, List <Tuple <DateTime, DateTime> > > response = await(Task <Tuple <Guid, List <Tuple <DateTime, DateTime> > > >) promise;
if (activations.Contains(response.Item1))
{
continue; // duplicate response from the same activation
}
activations.Add(response.Item1);
logger.Info(" {0}: Activation {1}", activations.Count, response.Item1);
int count = 1;
foreach (Tuple <DateTime, DateTime> call in response.Item2)
{
logger.Info("\t{0}: {1} - {2}", count++, LogFormatter.PrintDate(call.Item1), LogFormatter.PrintDate(call.Item2));
}
}
Assert.IsTrue(activations.Count <= ExpectedMaxLocalActivations, "activations.Count = " + activations.Count + " but expected no more than " + ExpectedMaxLocalActivations);
}
public static MongoSuspectTime Create(Tuple <SiloAddress, DateTime> tuple)
{
return(new MongoSuspectTime {
Address = tuple.Item1.ToParsableString(), IAmAliveTime = LogFormatter.PrintDate(tuple.Item2)
});
}
public async Task <bool> TryToSuspectOrKill(SiloAddress silo)
{
var table = await membershipTableProvider.ReadAll();
if (log.IsEnabled(LogLevel.Debug))
{
log.Debug("-TryToSuspectOrKill: Read Membership table {0}", table.ToString());
}
if (this.IsStopping)
{
this.log.LogInformation(
(int)ErrorCode.MembershipFoundMyselfDead3,
"Ignoring call to TrySuspectOrKill for silo {Silo} since the local silo is dead",
silo);
return(true);
}
var(localSiloEntry, _) = this.GetOrCreateLocalSiloEntry(table, this.CurrentStatus);
if (localSiloEntry.Status == SiloStatus.Dead)
{
var msg = string.Format("I should be Dead according to membership table (in TryToSuspectOrKill): entry = {0}.", localSiloEntry.ToFullString(full: true));
log.Warn(ErrorCode.MembershipFoundMyselfDead3, msg);
KillMyselfLocally(msg);
return(true);
}
if (!table.Contains(silo))
{
// this should not happen ...
var str = string.Format("-Could not find silo entry for silo {0} in the table.", silo);
log.Error(ErrorCode.MembershipFailedToReadSilo, str);
throw new KeyNotFoundException(str);
}
var tuple = table.Get(silo);
var entry = tuple.Item1.Copy();
string eTag = tuple.Item2;
if (log.IsEnabled(LogLevel.Debug))
{
log.Debug("-TryToSuspectOrKill {siloAddress}: The current status of {siloAddress} in the table is {status}, its entry is {entry}",
entry.SiloAddress, // First
entry.SiloAddress, // Second
entry.Status,
entry.ToFullString());
}
// check if the table already knows that this silo is dead
if (entry.Status == SiloStatus.Dead)
{
this.ProcessTableUpdate(table, "TrySuspectOrKill");
return(true);
}
var allVotes = entry.SuspectTimes ?? new List <Tuple <SiloAddress, DateTime> >();
// get all valid (non-expired) votes
var freshVotes = entry.GetFreshVotes(DateTime.UtcNow, this.clusterMembershipOptions.DeathVoteExpirationTimeout);
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("-Current number of fresh Voters for {0} is {1}", silo, freshVotes.Count.ToString());
}
if (freshVotes.Count >= this.clusterMembershipOptions.NumVotesForDeathDeclaration)
{
// this should not happen ...
var str = string.Format("-Silo {0} is suspected by {1} which is more or equal than {2}, but is not marked as dead. This is a bug!!!",
entry.SiloAddress, freshVotes.Count.ToString(), this.clusterMembershipOptions.NumVotesForDeathDeclaration.ToString());
log.Error(ErrorCode.Runtime_Error_100053, str);
KillMyselfLocally("Found a bug! Will stop.");
return(false);
}
// handle the corner case when the number of active silos is very small (then my only vote is enough)
int activeSilos = table.GetSiloStatuses(status => status == SiloStatus.Active, true, this.localSiloDetails.SiloAddress).Count;
// find if I have already voted
int myVoteIndex = freshVotes.FindIndex(voter => myAddress.Equals(voter.Item1));
// Try to kill:
// if there is NumVotesForDeathDeclaration votes (including me) to kill - kill.
// otherwise, if there is a majority of nodes (including me) voting to kill – kill.
bool declareDead = false;
int myAdditionalVote = myVoteIndex == -1 ? 1 : 0;
if (freshVotes.Count + myAdditionalVote >= this.clusterMembershipOptions.NumVotesForDeathDeclaration)
{
declareDead = true;
}
if (freshVotes.Count + myAdditionalVote >= (activeSilos + 1) / 2)
{
declareDead = true;
}
if (declareDead)
{
// kick this silo off
log.Info(ErrorCode.MembershipMarkingAsDead,
"-Going to mark silo {0} as DEAD in the table #1. I am the last voter: #freshVotes={1}, myVoteIndex = {2}, NumVotesForDeathDeclaration={3} , #activeSilos={4}, suspect list={5}",
entry.SiloAddress,
freshVotes.Count,
myVoteIndex,
this.clusterMembershipOptions.NumVotesForDeathDeclaration,
activeSilos,
PrintSuspectList(allVotes));
return(await DeclareDead(entry, eTag, table.Version));
}
// we still do not have enough votes - need to vote
// find voting place:
// update my vote, if I voted previously
// OR if the list is not full - just add a new vote
// OR overwrite the oldest entry.
int indexToWrite = allVotes.FindIndex(voter => myAddress.Equals(voter.Item1));
if (indexToWrite == -1)
{
// My vote is not recorded. Find the most outdated vote if the list is full, and overwrite it
if (allVotes.Count >= this.clusterMembershipOptions.NumVotesForDeathDeclaration) // if the list is full
{
// The list is full.
DateTime minVoteTime = allVotes.Min(voter => voter.Item2); // pick the most outdated vote
indexToWrite = allVotes.FindIndex(voter => voter.Item2.Equals(minVoteTime));
}
}
var prevList = allVotes.ToList(); // take a copy
var now = DateTime.UtcNow;
if (indexToWrite == -1)
{
// if did not find specific place to write (the list is not full), just add a new element to the list
entry.AddSuspector(myAddress, now);
}
else
{
var newEntry = new Tuple <SiloAddress, DateTime>(myAddress, now);
entry.SuspectTimes[indexToWrite] = newEntry;
}
log.Info(ErrorCode.MembershipVotingForKill,
"-Putting my vote to mark silo {0} as DEAD #2. Previous suspect list is {1}, trying to update to {2}, eTag={3}, freshVotes is {4}",
entry.SiloAddress,
PrintSuspectList(prevList),
PrintSuspectList(entry.SuspectTimes),
eTag,
PrintSuspectList(freshVotes));
// If we fail to update here we will retry later.
return(await membershipTableProvider.UpdateRow(entry, eTag, table.Version.Next()));
string PrintSuspectList(IEnumerable <Tuple <SiloAddress, DateTime> > list)
{
return(Utils.EnumerableToString(list, t => string.Format("<{0}, {1}>",
t.Item1, LogFormatter.PrintDate(t.Item2))));
}
}
/// <summary>
/// Handles cache purge signals
/// </summary>
/// <param name="lastItemPurged"></param>
/// <param name="newestItem"></param>
private void OnPurge(CachedMessage?lastItemPurged, CachedMessage?newestItem)
{
if (logger.IsEnabled(LogLevel.Debug) && lastItemPurged.HasValue && newestItem.HasValue)
{
logger.Debug($"CachePeriod: EnqueueTimeUtc: {LogFormatter.PrintDate(lastItemPurged.Value.EnqueueTimeUtc)} to {LogFormatter.PrintDate(newestItem.Value.EnqueueTimeUtc)}, DequeueTimeUtc: {LogFormatter.PrintDate(lastItemPurged.Value.DequeueTimeUtc)} to {LogFormatter.PrintDate(newestItem.Value.DequeueTimeUtc)}");
}
if (lastItemPurged.HasValue)
{
checkpointer.Update(this.dataAdapter.GetOffset(lastItemPurged.Value), DateTime.UtcNow);
}
}
private static async Task <T> ExecuteWithRetriesHelper <T>(
Func <int, Task <T> > function,
int callCounter,
int maxNumSuccessTries,
int maxNumErrorTries,
TimeSpan maxExecutionTime,
DateTime startExecutionTime,
Func <T, int, bool> retryValueFilter = null,
Func <Exception, int, bool> retryExceptionFilter = null,
IBackoffProvider onSuccessBackOff = null,
IBackoffProvider onErrorBackOff = null)
{
T result = default(T);
ExceptionDispatchInfo lastExceptionInfo = null;
bool retry;
do
{
retry = false;
if (maxExecutionTime != Constants.INFINITE_TIMESPAN && maxExecutionTime != default(TimeSpan))
{
DateTime now = DateTime.UtcNow;
if (now - startExecutionTime > maxExecutionTime)
{
if (lastExceptionInfo == null)
{
throw new TimeoutException(
$"ExecuteWithRetries has exceeded its max execution time of {maxExecutionTime}. Now is {LogFormatter.PrintDate(now)}, started at {LogFormatter.PrintDate(startExecutionTime)}, passed {now - startExecutionTime}");
}
lastExceptionInfo.Throw();
}
}
int counter = callCounter;
try
{
callCounter++;
result = await function(counter);
lastExceptionInfo = null;
if (callCounter < maxNumSuccessTries || maxNumSuccessTries == INFINITE_RETRIES) // -1 for infinite retries
{
if (retryValueFilter != null)
{
retry = retryValueFilter(result, counter);
}
}
if (retry)
{
TimeSpan?delay = onSuccessBackOff?.Next(counter);
if (delay.HasValue)
{
await Task.Delay(delay.Value);
}
}
}
catch (Exception exc)
{
retry = false;
if (callCounter < maxNumErrorTries || maxNumErrorTries == INFINITE_RETRIES)
{
if (retryExceptionFilter != null)
{
retry = retryExceptionFilter(exc, counter);
}
}
if (!retry)
{
throw;
}
lastExceptionInfo = ExceptionDispatchInfo.Capture(exc);
TimeSpan?delay = onErrorBackOff?.Next(counter);
if (delay.HasValue)
{
await Task.Delay(delay.Value);
}
}
} while (retry);
return(result);
}
public async Task <bool> TryToSuspectOrKill(SiloAddress silo)
{
var table = await membershipTableProvider.ReadAll();
var now = GetDateTimeUtcNow();
if (log.IsEnabled(LogLevel.Debug))
{
log.Debug("-TryToSuspectOrKill: Read Membership table {0}", table.ToString());
}
if (this.IsStopping)
{
this.log.LogInformation(
(int)ErrorCode.MembershipFoundMyselfDead3,
"Ignoring call to TrySuspectOrKill for silo {Silo} since the local silo is dead",
silo);
return(true);
}
var(localSiloEntry, _) = this.GetOrCreateLocalSiloEntry(table, this.CurrentStatus);
if (localSiloEntry.Status == SiloStatus.Dead)
{
var msg = string.Format("I should be Dead according to membership table (in TryToSuspectOrKill): entry = {0}.", localSiloEntry.ToFullString(full: true));
log.Warn(ErrorCode.MembershipFoundMyselfDead3, msg);
KillMyselfLocally(msg);
return(true);
}
if (!table.Contains(silo))
{
// this should not happen ...
var str = string.Format("-Could not find silo entry for silo {0} in the table.", silo);
log.Error(ErrorCode.MembershipFailedToReadSilo, str);
throw new KeyNotFoundException(str);
}
var tuple = table.Get(silo);
var entry = tuple.Item1.Copy();
string eTag = tuple.Item2;
if (log.IsEnabled(LogLevel.Debug))
{
log.Debug("-TryToSuspectOrKill {siloAddress}: The current status of {siloAddress} in the table is {status}, its entry is {entry}",
entry.SiloAddress, // First
entry.SiloAddress, // Second
entry.Status,
entry.ToFullString());
}
// Check if the table already knows that this silo is dead
if (entry.Status == SiloStatus.Dead)
{
this.ProcessTableUpdate(table, "TrySuspectOrKill");
return(true);
}
// Get all valid (non-expired) votes
var freshVotes = entry.GetFreshVotes(now, this.clusterMembershipOptions.DeathVoteExpirationTimeout);
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("-Current number of fresh Voters for {0} is {1}", silo, freshVotes.Count.ToString());
}
if (freshVotes.Count >= this.clusterMembershipOptions.NumVotesForDeathDeclaration)
{
// this should not happen ...
var str = string.Format("-Silo {0} is suspected by {1} which is more or equal than {2}, but is not marked as dead. This is a bug!!!",
entry.SiloAddress, freshVotes.Count.ToString(), this.clusterMembershipOptions.NumVotesForDeathDeclaration.ToString());
log.Error(ErrorCode.Runtime_Error_100053, str);
KillMyselfLocally("Found a bug! Will stop.");
return(false);
}
// Try to add our vote to the list and tally the fresh votes again.
var prevList = entry.SuspectTimes?.ToList() ?? new List <Tuple <SiloAddress, DateTime> >();
entry.AddOrUpdateSuspector(myAddress, now, clusterMembershipOptions.NumVotesForDeathDeclaration);
freshVotes = entry.GetFreshVotes(now, this.clusterMembershipOptions.DeathVoteExpirationTimeout);
// Determine if there are enough votes to evict the silo.
// Handle the corner case when the number of active silos is very small (then my only vote is enough)
int activeSilos = table.GetSiloStatuses(status => status == SiloStatus.Active, true, myAddress).Count;
if (freshVotes.Count >= clusterMembershipOptions.NumVotesForDeathDeclaration || freshVotes.Count >= (activeSilos + 1) / 2)
{
// Find the local silo's vote index
int myVoteIndex = freshVotes.FindIndex(voter => myAddress.Equals(voter.Item1));
// Kick this silo off
log.Info(ErrorCode.MembershipMarkingAsDead,
"-Going to mark silo {0} as DEAD in the table #1. This silo is the last voter: #FreshVotes={1}, MyVoteIndex = {2}, NumVotesForDeathDeclaration={3} , #activeSilos={4}, suspect list={5}",
entry.SiloAddress,
freshVotes.Count,
myVoteIndex,
this.clusterMembershipOptions.NumVotesForDeathDeclaration,
activeSilos,
PrintSuspectList(entry.SuspectTimes));
return(await DeclareDead(entry, eTag, table.Version, now));
}
log.Info(ErrorCode.MembershipVotingForKill,
"-Putting my vote to mark silo {0} as DEAD #2. Previous suspect list is {1}, trying to update to {2}, eTag={3}, freshVotes is {4}",
entry.SiloAddress,
PrintSuspectList(prevList),
PrintSuspectList(entry.SuspectTimes),
eTag,
PrintSuspectList(freshVotes));
// If we fail to update here we will retry later.
var ok = await membershipTableProvider.UpdateRow(entry, eTag, table.Version.Next());
if (ok)
{
table = await membershipTableProvider.ReadAll();
this.ProcessTableUpdate(table, "TrySuspectOrKill");
// Gossip using the local silo status, since this is just informational to propagate the suspicion vote.
GossipToOthers(localSiloEntry.SiloAddress, localSiloEntry.Status).Ignore();
}
return(ok);
string PrintSuspectList(IEnumerable <Tuple <SiloAddress, DateTime> > list)
{
return(Utils.EnumerableToString(list, t => string.Format("<{0}, {1}>",
t.Item1, LogFormatter.PrintDate(t.Item2))));
}
}
public async Task StatelessWorkerActivationsPerSiloDoNotExceedMaxLocalWorkersCount()
{
var gatewayOptions = this.Fixture.Client.ServiceProvider.GetRequiredService <IOptions <StaticGatewayListProviderOptions> >();
var gatewaysCount = gatewayOptions.Value.Gateways.Count;
// do extra calls to trigger activation of ExpectedMaxLocalActivations local activations
int numberOfCalls = ExpectedMaxLocalActivations * 3 * gatewaysCount;
IStatelessWorkerGrain grain = this.GrainFactory.GetGrain <IStatelessWorkerGrain>(GetRandomGrainId());
List <Task> promises = new List <Task>();
// warmup
for (int i = 0; i < gatewaysCount; i++)
{
promises.Add(grain.LongCall());
}
await Task.WhenAll(promises);
await Task.Delay(2000);
promises.Clear();
var stopwatch = Stopwatch.StartNew();
for (int i = 0; i < numberOfCalls; i++)
{
promises.Add(grain.LongCall());
}
await Task.WhenAll(promises);
stopwatch.Stop();
promises.Clear();
var statsTasks = new List <Task <Tuple <Guid, string, List <Tuple <DateTime, DateTime> > > > >();
for (int i = 0; i < numberOfCalls; i++)
{
statsTasks.Add(grain.GetCallStats()); // gather stats
}
await Task.WhenAll(promises);
var responsesPerSilo = statsTasks.Select(t => t.Result).GroupBy(s => s.Item2);
foreach (var siloGroup in responsesPerSilo)
{
var silo = siloGroup.Key;
HashSet <Guid> activations = new HashSet <Guid>();
foreach (var response in siloGroup)
{
if (activations.Contains(response.Item1))
{
continue; // duplicate response from the same activation
}
activations.Add(response.Item1);
output.WriteLine($"Silo {silo} with {activations.Count} activations: Activation {response.Item1}");
int count = 1;
foreach (Tuple <DateTime, DateTime> call in response.Item3)
{
output.WriteLine($"\t{count++}: {LogFormatter.PrintDate(call.Item1)} - {LogFormatter.PrintDate(call.Item2)}");
}
}
Assert.True(activations.Count <= ExpectedMaxLocalActivations, $"activations.Count = {activations.Count} in silo {silo} but expected no more than {ExpectedMaxLocalActivations}");
}
}
private static async Task <T> ExecuteWithRetriesHelper <T>(
Func <int, Task <T> > function,
int callCounter,
int maxNumSuccessTries,
int maxNumErrorTries,
TimeSpan maxExecutionTime,
DateTime startExecutionTime,
Func <T, int, bool> retryValueFilter = null,
Func <Exception, int, bool> retryExceptionFilter = null,
IBackoffProvider onSuccessBackOff = null,
IBackoffProvider onErrorBackOff = null)
{
if (maxExecutionTime != Constants.INFINITE_TIMESPAN && maxExecutionTime != default(TimeSpan))
{
DateTime now = DateTime.UtcNow;
if (now - startExecutionTime > maxExecutionTime)
{
Exception timeoutException = new TimeoutException(String.Format("ExecuteWithRetries has exceeded its max execution time of {0}. Now is {1}, started at {2}, passed {3}",
maxExecutionTime, LogFormatter.PrintDate(now), LogFormatter.PrintDate(startExecutionTime), now - startExecutionTime));
throw timeoutException;
}
}
T result = default(T);
int counter = callCounter;
Exception exception = null;
try
{
callCounter++;
result = await function(counter);
bool retry = false;
if (callCounter < maxNumSuccessTries || maxNumSuccessTries == INFINITE_RETRIES) // -1 for infinite retries
{
if (retryValueFilter != null)
{
retry = retryValueFilter(result, counter);
}
}
if (retry)
{
if (onSuccessBackOff == null)
{
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
else
{
TimeSpan delay = onSuccessBackOff.Next(counter);
await Task.Delay(delay);
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
}
return(result);
}
catch (Exception exc)
{
exception = exc;
}
if (exception != null)
{
bool retry = false;
if (callCounter < maxNumErrorTries || maxNumErrorTries == INFINITE_RETRIES)
{
if (retryExceptionFilter != null)
{
retry = retryExceptionFilter(exception, counter);
}
}
if (retry)
{
if (onErrorBackOff == null)
{
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
else
{
TimeSpan delay = onErrorBackOff.Next(counter);
await Task.Delay(delay);
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
}
ExceptionDispatchInfo.Capture(exception).Throw();
}
return(result); // this return value is just for the compiler to supress "not all control paths return a value".
}
public async Task UpdateIAmAlive(string deploymentId, SiloAddress address, DateTime iAmAliveTime)
{
await Collection.UpdateOneAsync(x => x.DeploymentId == deploymentId,
Update
.Set($"Members.{BuildKey(address)}.IAmAliveTime", LogFormatter.PrintDate(iAmAliveTime)));
}
public Task UpdateIAmAlive(string deploymentId, SiloAddress address, DateTime iAmAliveTime)
{
var id = ReturnId(deploymentId, address);
return(Collection.UpdateOneAsync(x => x.Id == id, Update.Set(x => x.IAmAliveTime, LogFormatter.PrintDate(iAmAliveTime))));
}
/// <summary>
/// Initialize this Orleans silo for execution with the specified Azure deploymentId
/// </summary>
/// <param name="config">If null, Config data will be read from silo config file as normal, otherwise use the specified config data.</param>
/// <param name="deploymentId">Azure DeploymentId this silo is running under</param>
/// <param name="connectionString">Azure DataConnectionString. If null, defaults to the DataConnectionString setting from the Azure configuration for this role.</param>
/// <returns><c>true</c> is the silo startup was successful</returns>
public bool Start(ClusterConfiguration config, string deploymentId = null, string connectionString = null)
{
// Program ident
Trace.TraceInformation("Starting {0} v{1}", this.GetType().FullName, RuntimeVersion.Current);
// Check if deployment id was specified
if (deploymentId == null)
{
deploymentId = serviceRuntimeWrapper.DeploymentId;
}
// Read endpoint info for this instance from Azure config
string instanceName = serviceRuntimeWrapper.InstanceName;
// Configure this Orleans silo instance
if (config == null)
{
host = new SiloHost(instanceName);
host.LoadOrleansConfig(); // Load config from file + Initializes logger configurations
}
else
{
host = new SiloHost(instanceName, config); // Use supplied config data + Initializes logger configurations
}
IPEndPoint myEndpoint = serviceRuntimeWrapper.GetIPEndpoint(SiloEndpointConfigurationKeyName);
IPEndPoint proxyEndpoint = serviceRuntimeWrapper.GetIPEndpoint(ProxyEndpointConfigurationKeyName);
host.SetSiloType(Silo.SiloType.Secondary);
int generation = SiloAddress.AllocateNewGeneration();
// Bootstrap this Orleans silo instance
myEntry = new SiloInstanceTableEntry
{
DeploymentId = deploymentId,
Address = myEndpoint.Address.ToString(),
Port = myEndpoint.Port.ToString(CultureInfo.InvariantCulture),
Generation = generation.ToString(CultureInfo.InvariantCulture),
HostName = host.Config.GetOrCreateNodeConfigurationForSilo(host.Name).DNSHostName,
ProxyPort = (proxyEndpoint != null ? proxyEndpoint.Port : 0).ToString(CultureInfo.InvariantCulture),
RoleName = serviceRuntimeWrapper.RoleName,
SiloName = instanceName,
UpdateZone = serviceRuntimeWrapper.UpdateDomain.ToString(CultureInfo.InvariantCulture),
FaultZone = serviceRuntimeWrapper.FaultDomain.ToString(CultureInfo.InvariantCulture),
StartTime = LogFormatter.PrintDate(DateTime.UtcNow),
PartitionKey = deploymentId,
RowKey = myEndpoint.Address + "-" + myEndpoint.Port + "-" + generation
};
if (connectionString == null)
{
connectionString = serviceRuntimeWrapper.GetConfigurationSettingValue(DataConnectionConfigurationSettingName);
}
try
{
siloInstanceManager = OrleansSiloInstanceManager.GetManager(
deploymentId, connectionString).WithTimeout(AzureTableDefaultPolicies.TableCreationTimeout).Result;
}
catch (Exception exc)
{
var error = String.Format("Failed to create OrleansSiloInstanceManager. This means CreateTableIfNotExist for silo instance table has failed with {0}",
LogFormatter.PrintException(exc));
Trace.TraceError(error);
logger.Error(ErrorCode.AzureTable_34, error, exc);
throw new OrleansException(error, exc);
}
// Always use Azure table for membership when running silo in Azure
host.SetSiloLivenessType(GlobalConfiguration.LivenessProviderType.AzureTable);
if (host.Config.Globals.ReminderServiceType == GlobalConfiguration.ReminderServiceProviderType.NotSpecified ||
host.Config.Globals.ReminderServiceType == GlobalConfiguration.ReminderServiceProviderType.ReminderTableGrain)
{
host.SetReminderServiceType(GlobalConfiguration.ReminderServiceProviderType.AzureTable);
}
host.SetExpectedClusterSize(serviceRuntimeWrapper.RoleInstanceCount);
siloInstanceManager.RegisterSiloInstance(myEntry);
// Initialise this Orleans silo instance
host.SetDeploymentId(deploymentId, connectionString);
host.SetSiloEndpoint(myEndpoint, generation);
host.SetProxyEndpoint(proxyEndpoint);
host.InitializeOrleansSilo();
logger.Info(ErrorCode.Runtime_Error_100288, "Successfully initialized Orleans silo '{0}' as a {1} node.", host.Name, host.Type);
return(StartSilo());
}
private static async Task <T> DoExecuteWithRetriesAsync <T>(
Func <int, Task <T> > userDefinedFunction,
int callCounter,
int maxNumSuccessTries,
int maxNumErrorTries,
TimeSpan maxExecutionTime,
DateTime startExecutionTime,
Func <T, int, bool> retryValueFilter = null,
Func <Exception, int, bool> retryExceptionFilter = null,
IBackOffProvider onSuccessBackOff = null,
IBackOffProvider onErrorBackOff = null)
{
onSuccessBackOff ??= DefaultBackOffProvider;
onErrorBackOff ??= DefaultBackOffProvider;
var result = default(T);
ExceptionDispatchInfo lastExceptionInfo = null;
bool needRetry;
do
{
needRetry = false;
if (maxExecutionTime != InfiniteTimespan &&
maxExecutionTime != default)
{
var now = DateTime.UtcNow;
if (now - startExecutionTime > maxExecutionTime)
{
if (lastExceptionInfo == null)
{
throw new TimeoutException(
$"ExecuteWithRetries has exceeded its max execution time of {maxExecutionTime}. Now is {LogFormatter.PrintDate(now)}, started at {LogFormatter.PrintDate(startExecutionTime)}, passed {now - startExecutionTime}");
}
lastExceptionInfo.Throw();
}
}
var counter = callCounter;
try
{
callCounter++;
result = await userDefinedFunction(counter);
lastExceptionInfo = null;
if (callCounter < maxNumSuccessTries ||
maxNumSuccessTries == InfiniteRetries) // -1 for infinite retries
{
if (retryValueFilter != null)
{
needRetry = retryValueFilter(result, counter);
}
}
if (needRetry)
{
var delay = onSuccessBackOff?.Next(counter);
if (delay.HasValue)
{
await Task.Delay(delay.Value);
}
}
}
catch (Exception exc)
{
needRetry = false;
if (callCounter < maxNumErrorTries ||
maxNumErrorTries == InfiniteRetries)
{
if (retryExceptionFilter != null)
{
needRetry = retryExceptionFilter(exc, counter);
}
}
if (!needRetry)
{
throw;
}
lastExceptionInfo = ExceptionDispatchInfo.Capture(exc);
var delay = onErrorBackOff?.Next(counter);
if (delay.HasValue)
{
await Task.Delay(delay.Value);
}
}
} while (needRetry);
return(result);
}