public void UpdateIAmAlive(MembershipEntry entry)
{
Tuple<MembershipEntry, string> data;
siloTable.TryGetValue(entry.SiloAddress, out data);
if (data == null) return;
data.Item1.IAmAliveTime = entry.IAmAliveTime;
siloTable[entry.SiloAddress] = new Tuple<MembershipEntry, string>(data.Item1, NewETag());
}
public Task<bool> UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion)
{
if (logger.IsVerbose) logger.Verbose("UpdateRow entry = {0}, etag = {1}, table version = {2}", entry.ToFullString(), etag, tableVersion);
bool result = table.Update(entry, etag, tableVersion);
if (result == false)
logger.Info(ErrorCode.MembershipGrainBasedTable3,
"Update of {0}, eTag {1}, table version {2} failed. Table now is {3}",
entry.ToFullString(), etag, tableVersion, table.ReadAll());
return Task.FromResult(result);
}
public Task<bool> InsertRow(MembershipEntry entry, TableVersion tableVersion)
{
if (logger.IsVerbose) logger.Verbose("InsertRow entry = {0}, table version = {1}", entry.ToFullString(), tableVersion);
bool result = table.Insert(entry, tableVersion);
if (result == false)
logger.Info(ErrorCode.MembershipGrainBasedTable2,
"Insert of {0} and table version {1} failed. Table now is {2}",
entry.ToFullString(), tableVersion, table.ReadAll());
return Task.FromResult(result);
}
public bool Update(MembershipEntry entry, string etag, TableVersion version)
{
Tuple<MembershipEntry, string> data;
siloTable.TryGetValue(entry.SiloAddress, out data);
if (data == null) return false;
if (!data.Item2.Equals(etag) || !tableVersion.VersionEtag.Equals(version.VersionEtag)) return false;
siloTable[entry.SiloAddress] = new Tuple<MembershipEntry, string>(
entry, lastETagCounter++.ToString(CultureInfo.InvariantCulture));
tableVersion = new TableVersion(version.Version, NewETag());
return true;
}
public async Task<bool> InsertRow(MembershipEntry entry, TableVersion tableVersion)
{
try
{
if (logger.IsVerbose) logger.Verbose("InsertRow entry = {0}, table version = {1}", entry.ToFullString(), tableVersion);
var tableEntry = Convert(entry, tableManager.DeploymentId);
var versionEntry = tableManager.CreateTableVersionEntry(tableVersion.Version);
bool result = await tableManager.InsertSiloEntryConditionally(
tableEntry, versionEntry, tableVersion.VersionEtag);
if (result == false)
logger.Warn(ErrorCode.AzureTable_22, String.Format("Insert failed due to contention on the table. Will retry. Entry {0}, table version = {1}", entry.ToFullString(), tableVersion));
return result;
}
catch (Exception exc)
{
logger.Warn(ErrorCode.AzureTable_23, String.Format("Intermediate error inserting entry {0} tableVersion {1} to the table {2}.",
entry.ToFullString(), (tableVersion == null ? "null" : tableVersion.ToString()), tableManager.TableName), exc);
throw;
}
}
private async Task <bool> CleanupMyTableEntries(MembershipTableData table)
{
if (this.IsStopping)
{
return(true);
}
var silosToDeclareDead = new List <Tuple <MembershipEntry, string> >();
foreach (var tuple in table.Members.Where(
tuple => tuple.Item1.SiloAddress.Endpoint.Equals(myAddress.Endpoint)))
{
var entry = tuple.Item1;
var siloAddress = entry.SiloAddress;
if (siloAddress.Generation.Equals(myAddress.Generation))
{
if (entry.Status == SiloStatus.Dead)
{
var msg = string.Format("I should be Dead according to membership table (in CleanupTableEntries): entry = {0}.", entry.ToFullString(full: true));
log.Warn(ErrorCode.MembershipFoundMyselfDead2, msg);
KillMyselfLocally(msg);
}
continue;
}
if (entry.Status == SiloStatus.Dead)
{
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("Skipping my previous old Dead entry in membership table: {0}", entry.ToFullString(full: true));
}
continue;
}
if (log.IsEnabled(LogLevel.Debug))
{
log.Debug("Temporal anomaly detected in membership table -- Me={0} Other me={1}",
myAddress, siloAddress);
}
// Temporal paradox - There is an older clone of this silo in the membership table
if (siloAddress.Generation < myAddress.Generation)
{
log.Warn(ErrorCode.MembershipDetectedOlder, "Detected older version of myself - Marking other older clone as Dead -- Current Me={0} Older Me={1}, Old entry= {2}",
myAddress, siloAddress, entry.ToFullString());
// Declare older clone of me as Dead.
silosToDeclareDead.Add(tuple); //return DeclareDead(entry, eTag, tableVersion);
}
else if (siloAddress.Generation > myAddress.Generation)
{
// I am the older clone - Newer version of me should survive - I need to kill myself
var msg = string.Format("Detected newer version of myself - I am the older clone so I will stop -- Current Me={0} Newer Me={1}, Current entry= {2}",
myAddress, siloAddress, entry.ToFullString());
log.Warn(ErrorCode.MembershipDetectedNewer, msg);
await this.UpdateStatus(SiloStatus.Dead);
KillMyselfLocally(msg);
return(true); // No point continuing!
}
}
if (silosToDeclareDead.Count == 0)
{
return(true);
}
if (log.IsEnabled(LogLevel.Debug))
{
log.Debug("CleanupTableEntries: About to DeclareDead {0} outdated silos in the table: {1}", silosToDeclareDead.Count,
Utils.EnumerableToString(silosToDeclareDead.Select(tuple => tuple.Item1), entry => entry.ToString()));
}
var result = true;
var nextVersion = table.Version;
foreach (var siloData in silosToDeclareDead)
{
MembershipEntry entry = siloData.Item1;
string eTag = siloData.Item2;
bool ok = await DeclareDead(entry, eTag, nextVersion);
if (!ok)
{
result = false;
}
nextVersion = nextVersion.Next(); // advance the table version (if write succeded, we advanced the version. if failed, someone else did. It is safe anyway).
}
return(result);
}
public Task <bool> UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion) => this.grain.UpdateRow(entry, etag, tableVersion);
/// <summary>
/// Updates membership row data.
/// </summary>
/// <param name="storage">The storage to use.</param>
/// <param name="query">The query to use.</param>
/// <param name="deploymentId">The deployment with which to insert row.</param>
/// <param name="etag">The etag of which to use to check if the membership data being updated is not stale.</param>
/// <param name="membershipEntry">The membership data to used to update database.</param>
/// <param name="version">The membership version used to update database.</param>
/// <returns><em>TRUE</em> if update SUCCEEDS. <em>FALSE</em> ot</returns>
internal static async Task<bool> UpdateMembershipRowAsync(this IRelationalStorage storage, string query, string deploymentId, string etag, MembershipEntry membershipEntry, TableVersion version)
{
var ret = await storage.ReadAsync(query, command =>
{
var direction = ParameterDirection.Input;
var siloIdParameter = CreateDeploymentIdParameter(command, deploymentId, direction);
command.Parameters.Add(siloIdParameter);
var versionParameter = CreateVersionParameter(command, version.Version, direction);
command.Parameters.Add(versionParameter);
var versionEtagParameter = CreateVersionEtagParameter(command, version.VersionEtag, direction);
command.Parameters.Add(versionEtagParameter);
//The insert membership row part.
var etagParameter = CreateEtagParameter(command, etag, direction);
command.Parameters.Add(etagParameter);
var addressParameter = CreateAddressParameter(command, membershipEntry.SiloAddress.Endpoint.Address, direction);
command.Parameters.Add(addressParameter);
var portParameter = CreatePortParameter(command, membershipEntry.SiloAddress.Endpoint.Port, direction);
command.Parameters.Add(portParameter);
var generationParameter = CreateGenerationParameter(command, membershipEntry.SiloAddress.Generation, direction);
command.Parameters.Add(generationParameter);
var hostNameParameter = CreateHostNameParameter(command, membershipEntry.HostName, direction);
command.Parameters.Add(hostNameParameter);
var statusParameter = CreateStatusParameter(command, membershipEntry.Status, direction);
command.Parameters.Add(statusParameter);
var proxyPortParameter = CreateProxyPortParameter(command, membershipEntry.ProxyPort, direction);
command.Parameters.Add(proxyPortParameter);
var roleNameParameter = CreateRoleNameParameter(command, membershipEntry.RoleName, direction);
command.Parameters.Add(roleNameParameter);
var instanceNameParameter = CreateInstanceNameParameter(command, membershipEntry.InstanceName, direction);
command.Parameters.Add(instanceNameParameter);
var updateZoneParameter = CreateUpdateZoneParameter(command, membershipEntry.UpdateZone, direction);
command.Parameters.Add(updateZoneParameter);
var faultZoneParameter = CreateFaultZoneParameter(command, membershipEntry.FaultZone, direction);
command.Parameters.Add(faultZoneParameter);
var startTimeParameter = CreateStartTimeParameter(command, membershipEntry.StartTime, direction);
command.Parameters.Add(startTimeParameter);
var iAmAliveTimeParameter = CreateIAmAliveTimeParameter(command, membershipEntry.IAmAliveTime, direction);
command.Parameters.Add(iAmAliveTimeParameter);
var suspectingSilosParameter = CreateSuspectingSilosParameter(command, membershipEntry, direction);
command.Parameters.Add(suspectingSilosParameter);
var suspectingTimesParameter = CreateSuspectingTimesParameter(command, membershipEntry, direction);
command.Parameters.Add(suspectingTimesParameter);
}, (selector, _) => { return selector.GetBoolean(0); }).ConfigureAwait(continueOnCapturedContext: false);
return ret.First();
}
private static void ConvertToRow(MembershipEntry memEntry, SqlCommand command, string deploymentId)
{
command.Parameters.Add(new SqlParameter {
ParameterName = "@deploymentid", DbType = DbType.String, Value = deploymentId
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@address", DbType = DbType.String, Value = memEntry.SiloAddress.Endpoint.Address.ToString()
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@port", DbType = DbType.Int32, Value = memEntry.SiloAddress.Endpoint.Port
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@generation", DbType = DbType.Int32, Value = memEntry.SiloAddress.Generation
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@hostname", DbType = DbType.String, Value = memEntry.HostName
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@status", DbType = DbType.Int32, Value = memEntry.Status
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@proxyport", DbType = DbType.Int32, Value = memEntry.ProxyPort
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@primary", DbType = DbType.Boolean, Value = memEntry.IsPrimary
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@rolename", DbType = DbType.String, Value = memEntry.RoleName
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@instancename", DbType = DbType.String, Value = memEntry.InstanceName
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@updatezone", DbType = DbType.Int32, Value = memEntry.UpdateZone
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@faultzone", DbType = DbType.Int32, Value = memEntry.FaultZone
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@starttime", DbType = DbType.DateTime, Value = memEntry.StartTime
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@iamalivetime", DbType = DbType.DateTime, Value = memEntry.IAmAliveTime
});
if (memEntry.SuspectTimes != null)
{
StringBuilder siloList = new StringBuilder();
StringBuilder timeList = new StringBuilder();
bool first = true;
foreach (var tuple in memEntry.SuspectTimes)
{
if (!first)
{
siloList.Append('|');
timeList.Append('|');
}
siloList.Append(tuple.Item1.ToParsableString());
timeList.Append(TraceLogger.PrintDate(tuple.Item2));
first = false;
}
command.Parameters.Add(new SqlParameter {
ParameterName = "@suspectingsilos", DbType = DbType.String, Value = siloList.ToString()
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@suspectingtimes", DbType = DbType.String, Value = timeList.ToString()
});
}
else
{
command.Parameters.Add(new SqlParameter {
ParameterName = "@suspectingsilos", DbType = DbType.String, Value = DBNull.Value
});
command.Parameters.Add(new SqlParameter {
ParameterName = "@suspectingtimes", DbType = DbType.String, Value = DBNull.Value
});
}
}
private static IDbDataParameter CreateSuspectingSilosParameter(IDbCommand command, MembershipEntry membershipEntry, ParameterDirection direction)
{
//TODO: Refactor the database to take DATETIME2(7) and change the data type here accordingly.
var parameter = command.CreateParameter();
parameter.ParameterName = "suspectingSilos";
parameter.DbType = DbType.String;
parameter.Direction = direction;
if(membershipEntry.SuspectTimes != null)
{
var siloList = new StringBuilder();
bool first = true;
foreach(var tuple in membershipEntry.SuspectTimes)
{
if(!first)
{
siloList.Append('|');
}
siloList.Append(tuple.Item1.ToParsableString());
first = false;
}
parameter.Value = siloList.ToString();
}
else
{
parameter.Value = DBNull.Value;
}
return parameter;
}
internal static ConsulSiloRegistration FromMembershipEntry(String deploymentId, MembershipEntry entry, String etag)
{
var ret = new ConsulSiloRegistration
{
DeploymentId = deploymentId,
Address = entry.SiloAddress,
IAmAliveTime = entry.IAmAliveTime,
LastIndex = Convert.ToUInt64(etag),
Hostname = entry.HostName,
ProxyPort = entry.ProxyPort,
StartTime = entry.StartTime,
Status = entry.Status,
SiloName = entry.SiloName,
SuspectingSilos = entry.SuspectTimes.Select(silo => new SuspectingSilo { Id = silo.Item1.ToParsableString(), Time = silo.Item2 }).ToList()
};
return ret;
}
private string Serialize(MembershipEntry value)
{
return(JsonConvert.SerializeObject(value, _jsonSerializerSettings));
}
protected async Task MembershipTable_ReadRow_Insert_Read(bool extendedProtocol = true)
{
MembershipTableData data = await membershipTable.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.Equal(0, data.Members.Count);
TableVersion newTableVersion = data.Version.Next();
MembershipEntry newEntry = CreateMembershipEntryForTest();
bool ok = await membershipTable.InsertRow(newEntry, newTableVersion);
Assert.True(ok, "InsertRow failed");
ok = await membershipTable.InsertRow(newEntry, newTableVersion);
Assert.False(ok, "InsertRow should have failed - same entry, old table version");
if (extendedProtocol)
{
ok = await membershipTable.InsertRow(CreateMembershipEntryForTest(), newTableVersion);
Assert.False(ok, "InsertRow should have failed - new entry, old table version");
}
data = await membershipTable.ReadAll();
if (extendedProtocol)
{
Assert.Equal(1, data.Version.Version);
}
TableVersion nextTableVersion = data.Version.Next();
ok = await membershipTable.InsertRow(newEntry, nextTableVersion);
Assert.False(ok, "InsertRow should have failed - duplicate entry");
data = await membershipTable.ReadAll();
Assert.Equal(1, data.Members.Count);
data = await membershipTable.ReadRow(newEntry.SiloAddress);
if (extendedProtocol)
{
Assert.Equal(newTableVersion.Version, data.Version.Version);
}
logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", data.Version, data);
Assert.Equal(1, data.Members.Count);
Assert.NotNull(data.Version.VersionEtag);
if (extendedProtocol)
{
Assert.NotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag);
Assert.Equal(newTableVersion.Version, data.Version.Version);
}
var membershipEntry = data.Members[0].Item1;
string eTag = data.Members[0].Item2;
logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} Entry={1}", eTag, membershipEntry);
Assert.NotNull(eTag);
Assert.NotNull(membershipEntry);
}
Task<bool> IMembershipTable.InsertRow(MembershipEntry entry, TableVersion tableVersion)
{
if (logger.IsVerbose3) logger.Verbose3(string.Format("SqlMembershipTable.InsertRow called with entry {0} and tableVersion {1}.", entry, tableVersion));
//The "tableVersion" parameter should always exist when inserting a row as Init should
//have been called and membership version created and read. This is an optimization to
//not to go through all the way to database to fail a conditional check on etag (which does
//exist for the sake of robustness) as mandated by Orleans membership protocol.
//Likewise, no update can be done without membership entry.
if (entry == null)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.InsertRow aborted due to null check. MembershipEntry is null.");
throw new ArgumentNullException("entry");
}
if (tableVersion == null)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.InsertRow aborted due to null check. TableVersion is null ");
throw new ArgumentNullException("tableVersion");
}
try
{
var query = queryConstants.GetConstant(database.InvariantName, QueryKeys.InsertMembershipKey);
return database.InsertMembershipRowAsync(query, deploymentId, entry, tableVersion);
}
catch(Exception ex)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.InsertRow failed: {0}", ex);
throw;
}
}
Task IMembershipTable.UpdateIAmAlive(MembershipEntry entry)
{
if(logger.IsVerbose3) logger.Verbose3(string.Format("IMembershipTable.UpdateIAmAlive called with entry {0}.", entry));
if (entry == null)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.UpdateIAmAlive aborted due to null check. MembershipEntry is null.");
throw new ArgumentNullException("entry");
}
try
{
var query = queryConstants.GetConstant(database.InvariantName, QueryKeys.UpdateIAmAlivetimeKey);
return database.UpdateIAmAliveTimeAsync(query, deploymentId, entry);
}
catch(Exception ex)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.UpdateIAmAlive failed: {0}", ex);
throw;
}
}
private static MembershipEntry Parse(SiloInstanceTableEntry tableEntry)
{
var parse = new MembershipEntry
{
HostName = tableEntry.HostName,
Status = (SiloStatus) Enum.Parse(typeof (SiloStatus), tableEntry.Status)
};
if (!string.IsNullOrEmpty(tableEntry.ProxyPort))
parse.ProxyPort = int.Parse(tableEntry.ProxyPort);
int port = 0;
if (!string.IsNullOrEmpty(tableEntry.Port))
int.TryParse(tableEntry.Port, out port);
int gen = 0;
if (!string.IsNullOrEmpty(tableEntry.Generation))
int.TryParse(tableEntry.Generation, out gen);
parse.SiloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Parse(tableEntry.Address), port), gen);
parse.RoleName = tableEntry.RoleName;
if (!string.IsNullOrEmpty(tableEntry.SiloName))
{
parse.SiloName = tableEntry.SiloName;
}else if (!string.IsNullOrEmpty(tableEntry.InstanceName))
{
// this is for backward compatability: in a mixed cluster of old and new version,
// some entries will have the old InstanceName column.
parse.SiloName = tableEntry.InstanceName;
}
if (!string.IsNullOrEmpty(tableEntry.UpdateZone))
parse.UpdateZone = int.Parse(tableEntry.UpdateZone);
if (!string.IsNullOrEmpty(tableEntry.FaultZone))
parse.FaultZone = int.Parse(tableEntry.FaultZone);
parse.StartTime = !string.IsNullOrEmpty(tableEntry.StartTime) ?
LogFormatter.ParseDate(tableEntry.StartTime) : default(DateTime);
parse.IAmAliveTime = !string.IsNullOrEmpty(tableEntry.IAmAliveTime) ?
LogFormatter.ParseDate(tableEntry.IAmAliveTime) : default(DateTime);
var suspectingSilos = new List<SiloAddress>();
var suspectingTimes = new List<DateTime>();
if (!string.IsNullOrEmpty(tableEntry.SuspectingSilos))
{
string[] silos = tableEntry.SuspectingSilos.Split('|');
foreach (string silo in silos)
{
suspectingSilos.Add(SiloAddress.FromParsableString(silo));
}
}
if (!string.IsNullOrEmpty(tableEntry.SuspectingTimes))
{
string[] times = tableEntry.SuspectingTimes.Split('|');
foreach (string time in times)
suspectingTimes.Add(LogFormatter.ParseDate(time));
}
if (suspectingSilos.Count != suspectingTimes.Count)
throw new OrleansException(String.Format("SuspectingSilos.Length of {0} as read from Azure table is not eqaul to SuspectingTimes.Length of {1}", suspectingSilos.Count, suspectingTimes.Count));
for (int i = 0; i < suspectingSilos.Count; i++)
parse.AddSuspector(suspectingSilos[i], suspectingTimes[i]);
return parse;
}
public async Task<bool> UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion)
{
try
{
if (logger.IsVerbose) logger.Verbose("UpdateRow entry = {0}, etag = {1}, table version = {2}", entry.ToFullString(), etag, tableVersion);
var siloEntry = Convert(entry, tableManager.DeploymentId);
var versionEntry = tableManager.CreateTableVersionEntry(tableVersion.Version);
bool result = await tableManager.UpdateSiloEntryConditionally(siloEntry, etag, versionEntry, tableVersion.VersionEtag);
if (result == false)
logger.Warn(ErrorCode.AzureTable_24,
$"Update failed due to contention on the table. Will retry. Entry {entry.ToFullString()}, eTag {etag}, table version = {tableVersion} ");
return result;
}
catch (Exception exc)
{
logger.Warn(ErrorCode.AzureTable_25,
$"Intermediate error updating entry {entry.ToFullString()} tableVersion {(tableVersion == null ? "null" : tableVersion.ToString())} to the table {tableManager.TableName}.", exc);
throw;
}
}
private static void UpdateIAmAliveTime(MembershipEntry memEntry, SqlCommand command, string deploymentId)
{
command.Parameters.Add(new SqlParameter { ParameterName = "@deploymentid", DbType = DbType.String, Value = deploymentId });
command.Parameters.Add(new SqlParameter { ParameterName = "@address", DbType = DbType.String, Value = memEntry.SiloAddress.Endpoint.Address.ToString() });
command.Parameters.Add(new SqlParameter { ParameterName = "@port", DbType = DbType.Int32, Value = memEntry.SiloAddress.Endpoint.Port });
command.Parameters.Add(new SqlParameter { ParameterName = "@generation", DbType = DbType.Int32, Value = memEntry.SiloAddress.Generation });
command.Parameters.Add(new SqlParameter { ParameterName = "@iamalivetime", DbType = DbType.DateTime, Value = memEntry.IAmAliveTime });
command.Parameters.Add(new SqlParameter { ParameterName = "@newetag", DbType = DbType.String, Value = NewEtag() });
}
private static MembershipEntry Parse(SiloInstanceTableEntry tableEntry)
{
var parse = new MembershipEntry
{
HostName = tableEntry.HostName,
Status = (SiloStatus)Enum.Parse(typeof(SiloStatus), tableEntry.Status)
};
if (!string.IsNullOrEmpty(tableEntry.ProxyPort))
{
parse.ProxyPort = int.Parse(tableEntry.ProxyPort);
}
int port = 0;
if (!string.IsNullOrEmpty(tableEntry.Port))
{
int.TryParse(tableEntry.Port, out port);
}
int gen = 0;
if (!string.IsNullOrEmpty(tableEntry.Generation))
{
int.TryParse(tableEntry.Generation, out gen);
}
parse.SiloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Parse(tableEntry.Address), port), gen);
parse.RoleName = tableEntry.RoleName;
if (!string.IsNullOrEmpty(tableEntry.SiloName))
{
parse.SiloName = tableEntry.SiloName;
}
else if (!string.IsNullOrEmpty(tableEntry.InstanceName))
{
// this is for backward compatability: in a mixed cluster of old and new version,
// some entries will have the old InstanceName column.
parse.SiloName = tableEntry.InstanceName;
}
if (!string.IsNullOrEmpty(tableEntry.UpdateZone))
{
parse.UpdateZone = int.Parse(tableEntry.UpdateZone);
}
if (!string.IsNullOrEmpty(tableEntry.FaultZone))
{
parse.FaultZone = int.Parse(tableEntry.FaultZone);
}
parse.StartTime = !string.IsNullOrEmpty(tableEntry.StartTime) ?
LogFormatter.ParseDate(tableEntry.StartTime) : default(DateTime);
parse.IAmAliveTime = !string.IsNullOrEmpty(tableEntry.IAmAliveTime) ?
LogFormatter.ParseDate(tableEntry.IAmAliveTime) : default(DateTime);
var suspectingSilos = new List <SiloAddress>();
var suspectingTimes = new List <DateTime>();
if (!string.IsNullOrEmpty(tableEntry.SuspectingSilos))
{
string[] silos = tableEntry.SuspectingSilos.Split('|');
foreach (string silo in silos)
{
suspectingSilos.Add(SiloAddress.FromParsableString(silo));
}
}
if (!string.IsNullOrEmpty(tableEntry.SuspectingTimes))
{
string[] times = tableEntry.SuspectingTimes.Split('|');
foreach (string time in times)
{
suspectingTimes.Add(LogFormatter.ParseDate(time));
}
}
if (suspectingSilos.Count != suspectingTimes.Count)
{
throw new OrleansException(String.Format("SuspectingSilos.Length of {0} as read from Azure table is not eqaul to SuspectingTimes.Length of {1}", suspectingSilos.Count, suspectingTimes.Count));
}
for (int i = 0; i < suspectingSilos.Count; i++)
{
parse.AddSuspector(suspectingSilos[i], suspectingTimes[i]);
}
return(parse);
}
public async Task <bool> InsertRow(MembershipEntry entry, TableVersion tableVersion)
{
return(await UpsertRowInternal(entry, tableVersion, updateTableVersion : true, allowInsertOnly : true) == UpsertResult.Success);
}
public Task UpdateIAmAlive(MembershipEntry entry)
{
return(manager.UpdateIAmAlive(entry));
}
public async Task UpdateIAmAlive(MembershipEntry entry)
{
var iAmAliveKV = ConsulSiloRegistrationAssembler.ToIAmAliveKVPair(_deploymentId, entry.SiloAddress, entry.IAmAliveTime);
await _consulClient.KV.Put(iAmAliveKV);
}
public async Task <bool> InsertRow(MembershipEntry entry, TableVersion tableVersion)
{
try
{
var siloEntity = this.ConvertToEntity(entry);
var versionEntity = this.BuildVersionEntity(tableVersion);
SiloEntity existentSiloEntry = default;
try
{
existentSiloEntry = ((JObject)await this._kubeClient.GetNamespacedCustomObjectAsync(
Constants.ORLEANS_GROUP,
Constants.PROVIDER_MODEL_VERSION,
this._namespace,
SiloEntity.PLURAL,
siloEntity.Metadata.Name
))?.ToObject <SiloEntity>();
}
catch (HttpOperationException ex)
{
if (ex.Response.StatusCode != HttpStatusCode.NotFound)
{
throw;
}
}
if (existentSiloEntry != null)
{
return(false);
}
var currentVersionEntity = await this.GetClusterVersion();
if (currentVersionEntity == null ||
currentVersionEntity.ClusterVersion == versionEntity.ClusterVersion)
{
return(false);
}
var updatedVersionEntity = await this._kubeClient.ReplaceNamespacedCustomObjectAsync(
versionEntity,
Constants.ORLEANS_GROUP,
Constants.PROVIDER_MODEL_VERSION,
this._namespace,
ClusterVersionEntity.PLURAL,
versionEntity.Metadata.Name
);
if (updatedVersionEntity == null)
{
return(false);
}
var createdSiloEntity = await this._kubeClient.CreateNamespacedCustomObjectAsync(
siloEntity,
Constants.ORLEANS_GROUP,
Constants.PROVIDER_MODEL_VERSION,
this._namespace,
SiloEntity.PLURAL
);
return(createdSiloEntity != null);
}
catch (InvalidOperationException)
{
return(false);
}
catch (Exception exc)
{
this._logger?.LogError(exc, "Unable to insert Silo Entry.");
throw;
}
}
public async Task UpdateIAmAlive(MembershipEntry entry)
{
var iAmAliveKV = ConsulSiloRegistrationAssembler.ToIAmAliveKVPair(this.clusterId, this.kvRootFolder, entry.SiloAddress, entry.IAmAliveTime);
await _consulClient.KV.Put(iAmAliveKV);
}
private static IDbDataParameter CreateSuspectingSilosParameter(IDbCommand command, MembershipEntry membershipEntry)
{
var parameter = command.CreateParameter();
parameter.ParameterName = "suspectingSilos";
parameter.DbType = DbType.String;
parameter.Direction = ParameterDirection.Input;
if(membershipEntry.SuspectTimes != null)
{
var siloList = new StringBuilder();
bool first = true;
foreach(var tuple in membershipEntry.SuspectTimes)
{
if(!first)
{
siloList.Append('|');
}
siloList.Append(tuple.Item1.ToParsableString());
first = false;
}
parameter.Value = siloList.ToString();
}
else
{
parameter.Value = DBNull.Value;
}
return parameter;
}
public Task <bool> InsertRow(MembershipEntry entry, TableVersion tableVersion)
{
return(Task.FromResult(true));
}
private static IDbDataParameter CreateSuspectingTimesParameter(IDbCommand command, MembershipEntry membershipEntry, ParameterDirection direction)
{
var parameter = command.CreateParameter();
parameter.ParameterName = "suspectingTimes";
parameter.DbType = DbType.String;
parameter.Direction = direction;
if(membershipEntry.SuspectTimes != null)
{
var timeList = new StringBuilder();
bool first = true;
foreach(var tuple in membershipEntry.SuspectTimes)
{
if(!first)
{
timeList.Append('|');
}
timeList.Append(TraceLogger.PrintDate(tuple.Item2));
first = false;
}
parameter.Value = timeList.ToString();
}
else
{
parameter.Value = DBNull.Value;
}
return parameter;
}
public Task UpdateIAmAlive(MembershipEntry entry)
{
return(Task.CompletedTask);
}
Task<bool> IMembershipTable.UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion)
{
if (logger.IsVerbose3) logger.Verbose3(string.Format("IMembershipTable.UpdateRow called with entry {0}, etag {1} and tableVersion {2}.", entry, etag, tableVersion));
//The "tableVersion" parameter should always exist when updating a row as Init should
//have been called and membership version created and read. This is an optimization to
//not to go through all the way to database to fail a conditional check (which does
//exist for the sake of robustness) as mandated by Orleans membership protocol.
//Likewise, no update can be done without membership entry or an etag.
if (entry == null)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.UpdateRow aborted due to null check. MembershipEntry is null.");
throw new ArgumentNullException("entry");
}
if (etag == null)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.UpdateRow aborted due to null check. etag is null.");
throw new ArgumentNullException("etag");
}
if (tableVersion == null)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.UpdateRow aborted due to null check. TableVersion is null ");
throw new ArgumentNullException("tableVersion");
}
try
{
return orleansQueries.UpdateMembershipRowAsync(deploymentId, etag, entry, tableVersion);
}
catch(Exception ex)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.UpdateRow failed: {0}", ex);
throw;
}
}
public Task <bool> UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion)
{
return(Task.FromResult(true));
}
internal static ConsulSiloRegistration FromMembershipEntry(String deploymentId, MembershipEntry entry, String etag)
{
var ret = new ConsulSiloRegistration
{
DeploymentId = deploymentId,
Address = entry.SiloAddress,
IAmAliveTime = entry.IAmAliveTime,
LastIndex = Convert.ToUInt64(etag),
Hostname = entry.HostName,
ProxyPort = entry.ProxyPort,
StartTime = entry.StartTime,
Status = entry.Status,
SuspectingSilos = entry.SuspectTimes.Select(silo => new SuspectingSilo {
Id = silo.Item1.ToParsableString(), Time = silo.Item2
}).ToList()
};
return(ret);
}
public async Task UpdateIAmAlive(MembershipEntry entry)
{
try
{
if (logger.IsVerbose) logger.Verbose("Merge entry = {0}", entry.ToFullString());
var siloEntry = ConvertPartial(entry, tableManager.DeploymentId);
await tableManager.MergeTableEntryAsync(siloEntry);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.AzureTable_26, String.Format("Intermediate error updating IAmAlive field for entry {0} to the table {1}.", entry.ToFullString(), tableManager.TableName), exc);
throw;
}
}
public Task <bool> InsertRow(MembershipEntry entry, TableVersion tableVersion) => this.grain.InsertRow(entry, tableVersion);
private static MembershipEntry Parse(SiloInstanceTableEntry tableEntry)
{
var parse = new MembershipEntry
{
HostName = tableEntry.HostName,
Status = (SiloStatus) Enum.Parse(typeof (SiloStatus), tableEntry.Status)
};
if (!string.IsNullOrEmpty(tableEntry.ProxyPort))
parse.ProxyPort = int.Parse(tableEntry.ProxyPort);
parse.IsPrimary = false; // there are no primaries with in Azure table.
int port = 0;
if (!string.IsNullOrEmpty(tableEntry.Port))
int.TryParse(tableEntry.Port, out port);
int gen = 0;
if (!string.IsNullOrEmpty(tableEntry.Generation))
int.TryParse(tableEntry.Generation, out gen);
parse.SiloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Parse(tableEntry.Address), port), gen);
parse.RoleName = tableEntry.RoleName;
parse.InstanceName = tableEntry.InstanceName;
if (!string.IsNullOrEmpty(tableEntry.UpdateZone))
parse.UpdateZone = int.Parse(tableEntry.UpdateZone);
if (!string.IsNullOrEmpty(tableEntry.FaultZone))
parse.FaultZone = int.Parse(tableEntry.FaultZone);
parse.StartTime = !string.IsNullOrEmpty(tableEntry.StartTime) ?
TraceLogger.ParseDate(tableEntry.StartTime) : default(DateTime);
parse.IAmAliveTime = !string.IsNullOrEmpty(tableEntry.IAmAliveTime) ?
TraceLogger.ParseDate(tableEntry.IAmAliveTime) : default(DateTime);
var suspectingSilos = new List<SiloAddress>();
var suspectingTimes = new List<DateTime>();
if (!string.IsNullOrEmpty(tableEntry.SuspectingSilos))
{
string[] silos = tableEntry.SuspectingSilos.Split('|');
foreach (string silo in silos)
{
suspectingSilos.Add(SiloAddress.FromParsableString(silo));
}
}
if (!string.IsNullOrEmpty(tableEntry.SuspectingTimes))
{
string[] times = tableEntry.SuspectingTimes.Split('|');
foreach (string time in times)
suspectingTimes.Add(TraceLogger.ParseDate(time));
}
if (suspectingSilos.Count != suspectingTimes.Count)
throw new OrleansException(String.Format("SuspectingSilos.Length of {0} as read from Azure table is not eqaul to SuspectingTimes.Length of {1}", suspectingSilos.Count, suspectingTimes.Count));
for (int i = 0; i < suspectingSilos.Count; i++)
parse.AddSuspector(suspectingSilos[i], suspectingTimes[i]);
return parse;
}
public Task UpdateIAmAlive(MembershipEntry entry) => this.grain.UpdateIAmAlive(entry);
public async Task UpdateIAmAlive(MembershipEntry entry)
{
await PutMembershipEntry(entry, null);
}
public Task <bool> InsertRow(MembershipEntry entry, TableVersion tableVersion)
{
return(manager.InsertRow(entry, tableVersion));
}
public VersionedEntry(MembershipEntry entry, TableVersion tableVersion)
{
Entry = entry;
TableVersion = tableVersion;
ResourceVersion = tableVersion.VersionEtag;
}
public Task <bool> UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion)
{
return(manager.UpdateRow(entry, tableVersion, etag));
}
public async Task <bool> UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion)
{
try
{
if (this.logger.IsEnabled(LogLevel.Debug))
{
this.logger.Debug("UpdateRow entry = {0}, etag = {1}", entry.ToFullString(), etag);
}
var siloEntry = Convert(entry, tableVersion);
if (!int.TryParse(etag, out var currentEtag))
{
this.logger.Warn(ErrorCode.MembershipBase,
$"Update failed. Invalid ETag value. Will retry. Entry {entry.ToFullString()}, eTag {etag}");
return(false);
}
siloEntry.ETag = currentEtag + 1;
if (!TryCreateTableVersionRecord(tableVersion.Version, tableVersion.VersionEtag, out var versionEntry))
{
this.logger.Warn(ErrorCode.MembershipBase,
$"Update failed. Invalid ETag value. Will retry. Entry {entry.ToFullString()}, eTag {tableVersion.VersionEtag}");
return(false);
}
versionEntry.ETag++;
bool result;
try
{
var etagConditionalExpression = $"{SiloInstanceRecord.ETAG_PROPERTY_NAME} = {CURRENT_ETAG_ALIAS}";
var siloConditionalValues = new Dictionary <string, AttributeValue> {
{ CURRENT_ETAG_ALIAS, new AttributeValue {
N = etag
} }
};
var siloEntryUpdate = new Update
{
TableName = this.options.TableName,
Key = siloEntry.GetKeys(),
ConditionExpression = etagConditionalExpression
};
(siloEntryUpdate.UpdateExpression, siloEntryUpdate.ExpressionAttributeValues) =
this.storage.ConvertUpdate(siloEntry.GetFields(), siloConditionalValues);
var versionConditionalValues = new Dictionary <string, AttributeValue> {
{ CURRENT_ETAG_ALIAS, new AttributeValue {
N = tableVersion.VersionEtag
} }
};
var versionEntryUpdate = new Update
{
TableName = this.options.TableName,
Key = versionEntry.GetKeys(),
ConditionExpression = etagConditionalExpression
};
(versionEntryUpdate.UpdateExpression, versionEntryUpdate.ExpressionAttributeValues) =
this.storage.ConvertUpdate(versionEntry.GetFields(), versionConditionalValues);
await this.storage.WriteTxAsync(updates : new[] { siloEntryUpdate, versionEntryUpdate });
result = true;
}
catch (TransactionCanceledException canceledException)
{
if (canceledException.Message.Contains("ConditionalCheckFailed")) //not a good way to check for this currently
{
result = false;
this.logger.Warn(ErrorCode.MembershipBase,
$"Update failed due to contention on the table. Will retry. Entry {entry.ToFullString()}, eTag {etag}");
}
else
{
throw;
}
}
return(result);
}
catch (Exception exc)
{
this.logger.Warn(ErrorCode.MembershipBase,
$"Intermediate error updating entry {entry.ToFullString()} to the table {this.options.TableName}.", exc);
throw;
}
}
private static SiloInstanceTableEntry Convert(MembershipEntry memEntry, string deploymentId)
{
var tableEntry = new SiloInstanceTableEntry
{
DeploymentId = deploymentId,
Address = memEntry.SiloAddress.Endpoint.Address.ToString(),
Port = memEntry.SiloAddress.Endpoint.Port.ToString(CultureInfo.InvariantCulture),
Generation = memEntry.SiloAddress.Generation.ToString(CultureInfo.InvariantCulture),
HostName = memEntry.HostName,
Status = memEntry.Status.ToString(),
ProxyPort = memEntry.ProxyPort.ToString(CultureInfo.InvariantCulture),
RoleName = memEntry.RoleName,
InstanceName = memEntry.InstanceName,
UpdateZone = memEntry.UpdateZone.ToString(CultureInfo.InvariantCulture),
FaultZone = memEntry.FaultZone.ToString(CultureInfo.InvariantCulture),
StartTime = TraceLogger.PrintDate(memEntry.StartTime),
IAmAliveTime = TraceLogger.PrintDate(memEntry.IAmAliveTime)
};
if (memEntry.SuspectTimes != null)
{
var siloList = new StringBuilder();
var timeList = new StringBuilder();
bool first = true;
foreach (var tuple in memEntry.SuspectTimes)
{
if (!first)
{
siloList.Append('|');
timeList.Append('|');
}
siloList.Append(tuple.Item1.ToParsableString());
timeList.Append(TraceLogger.PrintDate(tuple.Item2));
first = false;
}
tableEntry.SuspectingSilos = siloList.ToString();
tableEntry.SuspectingTimes = timeList.ToString();
}
else
{
tableEntry.SuspectingSilos = String.Empty;
tableEntry.SuspectingTimes = String.Empty;
}
tableEntry.PartitionKey = deploymentId;
tableEntry.RowKey = SiloInstanceTableEntry.ConstructRowKey(memEntry.SiloAddress);
return tableEntry;
}
private async Task <bool> UpsertRow(MembershipEntry entry, TableVersion tableVersion, string checkEtag)
{
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.Trace(
$"RedisMembershipTable.UpsertRow called with entry {entry} and tableVersion {tableVersion}.");
}
if (entry == null)
{
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.Debug(
"RedisMembershipTable.UpsertRow aborted due to null check. MembershipEntry is null.");
}
throw new ArgumentNullException(nameof(entry));
}
if (tableVersion == null)
{
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.Debug("RedisMembershipTable.UpsertRow aborted due to null check. TableVersion is null ");
}
throw new ArgumentNullException(nameof(tableVersion));
}
var hashKeyData = $"{entry.SiloAddress.ToParsableString()}-data";
var hashKeyAlive = $"{entry.SiloAddress.ToParsableString()}-alive";
var hashKeyEtag = $"{entry.SiloAddress.ToParsableString()}-etag";
var data = JsonConvert.SerializeObject(entry, SerializationSettings.Value);
var alive = JsonConvert.SerializeObject(entry.IAmAliveTime, SerializationSettings.Value);
var etag = Guid.NewGuid().ToString();
try
{
var txn = _db.CreateTransaction();
// async calls in Redis Transaction do not have to be awaited
#pragma warning disable 4014
txn.AddCondition(Condition.HashEqual(_clusterKey, TableVersionEtagKey, tableVersion.VersionEtag));
txn.AddCondition(checkEtag == null
? Condition.HashNotExists(_clusterKey, hashKeyData)
: Condition.HashEqual(_clusterKey, hashKeyEtag, checkEtag));
txn.HashSetAsync(_clusterKey, new[]
{
new HashEntry(TableVersionKey, tableVersion.Version),
new HashEntry(TableVersionEtagKey, etag),
new HashEntry(hashKeyData, data),
new HashEntry(hashKeyAlive, alive),
new HashEntry(hashKeyEtag, etag)
});
#pragma warning restore 4014
return(await txn.ExecuteAsync());
}
catch (Exception ex)
{
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.Debug("RedisMembershipTable.UpsertRow failed: {0}", ex);
}
throw;
}
}
public Task UpdateIAmAlive(MembershipEntry entry)
{
if (logger.IsVerbose) logger.Verbose("UpdateIAmAlive entry = {0}", entry.ToFullString());
table.UpdateIAmAlive(entry);
return TaskDone.Done;
}
private static SiloInstanceTableEntry ConvertPartial(MembershipEntry memEntry, string deploymentId)
{
return new SiloInstanceTableEntry
{
DeploymentId = deploymentId,
IAmAliveTime = TraceLogger.PrintDate(memEntry.IAmAliveTime),
PartitionKey = deploymentId,
RowKey = SiloInstanceTableEntry.ConstructRowKey(memEntry.SiloAddress)
};
}
public async Task <bool> InsertRow(MembershipEntry entry, TableVersion tableVersion)
{
try
{
if (this.logger.IsEnabled(LogLevel.Debug))
{
this.logger.Debug("InsertRow entry = {0}", entry.ToFullString());
}
var tableEntry = Convert(entry, tableVersion);
if (!TryCreateTableVersionRecord(tableVersion.Version, tableVersion.VersionEtag, out var versionEntry))
{
this.logger.Warn(ErrorCode.MembershipBase,
$"Insert failed. Invalid ETag value. Will retry. Entry {entry.ToFullString()}, eTag {tableVersion.VersionEtag}");
return(false);
}
versionEntry.ETag++;
bool result;
try
{
var notExistConditionExpression =
$"attribute_not_exists({SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME}) AND attribute_not_exists({SiloInstanceRecord.SILO_IDENTITY_PROPERTY_NAME})";
var tableEntryInsert = new Put
{
Item = tableEntry.GetFields(true),
ConditionExpression = notExistConditionExpression,
TableName = this.options.TableName
};
var conditionalValues = new Dictionary <string, AttributeValue> {
{ CURRENT_ETAG_ALIAS, new AttributeValue {
N = tableVersion.VersionEtag
} }
};
var etagConditionalExpression = $"{SiloInstanceRecord.ETAG_PROPERTY_NAME} = {CURRENT_ETAG_ALIAS}";
var versionEntryUpdate = new Update
{
TableName = this.options.TableName,
Key = versionEntry.GetKeys(),
ConditionExpression = etagConditionalExpression
};
(versionEntryUpdate.UpdateExpression, versionEntryUpdate.ExpressionAttributeValues) =
this.storage.ConvertUpdate(versionEntry.GetFields(), conditionalValues);
await this.storage.WriteTxAsync(new[] { tableEntryInsert }, new[] { versionEntryUpdate });
result = true;
}
catch (TransactionCanceledException canceledException)
{
if (canceledException.Message.Contains("ConditionalCheckFailed")) //not a good way to check for this currently
{
result = false;
this.logger.Warn(ErrorCode.MembershipBase,
$"Insert failed due to contention on the table. Will retry. Entry {entry.ToFullString()}");
}
else
{
throw;
}
}
return(result);
}
catch (Exception exc)
{
this.logger.Warn(ErrorCode.MembershipBase,
$"Intermediate error inserting entry {entry.ToFullString()} to the table {this.options.TableName}.", exc);
throw;
}
}
private static MembershipEntry Parse(SiloInstanceTableEntry tableEntry)
{
var parse = new MembershipEntry
{
HostName = tableEntry.HostName,
Status = (SiloStatus)Enum.Parse(typeof(SiloStatus), tableEntry.Status)
};
if (!string.IsNullOrEmpty(tableEntry.ProxyPort))
{
parse.ProxyPort = int.Parse(tableEntry.ProxyPort);
}
parse.IsPrimary = false; // there are no primaries with in Azure table.
int port = 0;
if (!string.IsNullOrEmpty(tableEntry.Port))
{
int.TryParse(tableEntry.Port, out port);
}
int gen = 0;
if (!string.IsNullOrEmpty(tableEntry.Generation))
{
int.TryParse(tableEntry.Generation, out gen);
}
parse.SiloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Parse(tableEntry.Address), port), gen);
parse.RoleName = tableEntry.RoleName;
parse.InstanceName = tableEntry.InstanceName;
if (!string.IsNullOrEmpty(tableEntry.UpdateZone))
{
parse.UpdateZone = int.Parse(tableEntry.UpdateZone);
}
if (!string.IsNullOrEmpty(tableEntry.FaultZone))
{
parse.FaultZone = int.Parse(tableEntry.FaultZone);
}
parse.StartTime = !string.IsNullOrEmpty(tableEntry.StartTime) ?
TraceLogger.ParseDate(tableEntry.StartTime) : default(DateTime);
parse.IAmAliveTime = !string.IsNullOrEmpty(tableEntry.IAmAliveTime) ?
TraceLogger.ParseDate(tableEntry.IAmAliveTime) : default(DateTime);
var suspectingSilos = new List <SiloAddress>();
var suspectingTimes = new List <DateTime>();
if (!string.IsNullOrEmpty(tableEntry.SuspectingSilos))
{
string[] silos = tableEntry.SuspectingSilos.Split('|');
foreach (string silo in silos)
{
suspectingSilos.Add(SiloAddress.FromParsableString(silo));
}
}
if (!string.IsNullOrEmpty(tableEntry.SuspectingTimes))
{
string[] times = tableEntry.SuspectingTimes.Split('|');
foreach (string time in times)
{
suspectingTimes.Add(TraceLogger.ParseDate(time));
}
}
if (suspectingSilos.Count != suspectingTimes.Count)
{
throw new OrleansException(String.Format("SuspectingSilos.Length of {0} as read from Azure table is not eqaul to SuspectingTimes.Length of {1}", suspectingSilos.Count, suspectingTimes.Count));
}
for (int i = 0; i < suspectingSilos.Count; i++)
{
parse.AddSuspector(suspectingSilos[i], suspectingTimes[i]);
}
return(parse);
}
public async Task UpdateIAmAlive(MembershipEntry entry)
{
var key = this.CreateSiloKey(this.deploymentId, entry.SiloAddress.ToString());
await this.firebaseClient.PutAsync(this.ConstructMembershipPath($"{key}/{SiloInstanceRecord.IAmAliveTimePropertName}"), entry.IAmAliveTime);
}
public async Task GatewaySelection_SqlServer()
{
string testName = TestContext.TestName;
Console.WriteLine(TestUtils.DumpTestContext(TestContext));
Guid serviceId = Guid.NewGuid();
GlobalConfiguration cfg = new GlobalConfiguration
{
ServiceId = serviceId,
DeploymentId = testName,
DataConnectionString = TestHelper.TestUtils.GetSqlConnectionString(TestContext)
};
var membership = new SqlMembershipTable();
var logger = TraceLogger.GetLogger(membership.GetType().Name);
await membership.InitializeMembershipTable(cfg, true, logger);
IMembershipTable membershipTable = membership;
// Pre-populate gateway table with data
int count = 1;
foreach (Uri gateway in gatewayAddressUris)
{
Console.WriteLine("Adding gataway data for {0}", gateway);
SiloAddress siloAddress = gateway.ToSiloAddress();
Assert.IsNotNull(siloAddress, "Unable to get SiloAddress from Uri {0}", gateway);
MembershipEntry MembershipEntry = new MembershipEntry
{
SiloAddress = siloAddress,
HostName = gateway.Host,
Status = SiloStatus.Active,
ProxyPort = gateway.Port,
StartTime = DateTime.UtcNow
};
var tableVersion = new TableVersion(count, Guid.NewGuid().ToString());
Console.WriteLine("Inserting gataway data for {0} with TableVersion={1}", MembershipEntry, tableVersion);
bool ok = await membershipTable.InsertRow(MembershipEntry, tableVersion);
count++;
Assert.IsTrue(ok, "Membership record should have been written OK but were not: {0}", MembershipEntry);
Console.WriteLine("Successfully inserted Membership row {0}", MembershipEntry);
}
MembershipTableData data = await membershipTable.ReadAll();
Assert.IsNotNull(data, "MembershipTableData returned");
Assert.AreEqual(gatewayAddressUris.Count, data.Members.Count, "Number of gateway records read");
IGatewayListProvider listProvider = membership;
Test_GatewaySelection(listProvider);
}
/// <summary>
/// Updates membership row data.
/// </summary>
/// <param name="deploymentId">The deployment with which to insert row.</param>
/// <param name="etag">The etag of which to use to check if the membership data being updated is not stale.</param>
/// <param name="membershipEntry">The membership data to used to update database.</param>
/// <param name="version">The membership version used to update database.</param>
/// <returns><em>TRUE</em> if update SUCCEEDS. <em>FALSE</em> ot</returns>
internal async Task<bool> UpdateMembershipRowAsync(string deploymentId, string etag, MembershipEntry membershipEntry, TableVersion version)
{
var ret = await storage.ReadAsync(orleansQueries.UpdateMembershipKey, command =>
{
var siloIdParameter = CreateDeploymentIdParameter(command, deploymentId);
command.Parameters.Add(siloIdParameter);
var versionEtagParameter = CreateVersionEtagParameter(command, version.VersionEtag);
command.Parameters.Add(versionEtagParameter);
//The insert membership row part.
var etagParameter = CreateEtagParameter(command, etag);
command.Parameters.Add(etagParameter);
var addressParameter = CreateAddressParameter(command, membershipEntry.SiloAddress.Endpoint.Address);
command.Parameters.Add(addressParameter);
var portParameter = CreatePortParameter(command, membershipEntry.SiloAddress.Endpoint.Port);
command.Parameters.Add(portParameter);
var generationParameter = CreateGenerationParameter(command, membershipEntry.SiloAddress.Generation);
command.Parameters.Add(generationParameter);
var hostNameParameter = CreateHostNameParameter(command, membershipEntry.HostName);
command.Parameters.Add(hostNameParameter);
var statusParameter = CreateStatusParameter(command, membershipEntry.Status);
command.Parameters.Add(statusParameter);
var proxyPortParameter = CreateProxyPortParameter(command, membershipEntry.ProxyPort);
command.Parameters.Add(proxyPortParameter);
var roleNameParameter = CreateRoleNameParameter(command, membershipEntry.RoleName);
command.Parameters.Add(roleNameParameter);
var instanceNameParameter = CreateInstanceNameParameter(command, membershipEntry.InstanceName);
command.Parameters.Add(instanceNameParameter);
var updateZoneParameter = CreateUpdateZoneParameter(command, membershipEntry.UpdateZone);
command.Parameters.Add(updateZoneParameter);
var faultZoneParameter = CreateFaultZoneParameter(command, membershipEntry.FaultZone);
command.Parameters.Add(faultZoneParameter);
var startTimeParameter = CreateStartTimeParameter(command, membershipEntry.StartTime);
command.Parameters.Add(startTimeParameter);
var iAmAliveTimeParameter = CreateIAmAliveTimeParameter(command, membershipEntry.IAmAliveTime);
command.Parameters.Add(iAmAliveTimeParameter);
var suspectingSilosParameter = CreateSuspectingSilosParameter(command, membershipEntry);
command.Parameters.Add(suspectingSilosParameter);
var suspectingTimesParameter = CreateSuspectingTimesParameter(command, membershipEntry);
command.Parameters.Add(suspectingTimesParameter);
}, (selector, resultSetCount, cancellationToken) => { return Task.FromResult(selector.GetBoolean(0)); }).ConfigureAwait(continueOnCapturedContext: false);
return ret.First();
}
internal void UpdateMyFaultAndUpdateZone(MembershipEntry entry)
{
this.myFaultAndUpdateZones = new UpdateFaultCombo(entry.FaultZone, entry.UpdateZone);
}
/// <summary>
/// Updates IAmAlive for a silo
/// </summary>
/// <param name="deploymentId"></param>
/// <param name="membershipEntry"></param>
/// <returns></returns>
internal async Task UpdateIAmAliveTimeAsync(string deploymentId, MembershipEntry membershipEntry)
{
await storage.ExecuteAsync(orleansQueries.UpdateIAmAlivetimeKey, command =>
{
var siloIdParameter = CreateDeploymentIdParameter(command, deploymentId);
command.Parameters.Add(siloIdParameter);
var iAmAliveTimeParameter = CreateIAmAliveTimeParameter(command, membershipEntry.IAmAliveTime);
command.Parameters.Add(iAmAliveTimeParameter);
var addressParameter = CreateAddressParameter(command, membershipEntry.SiloAddress.Endpoint.Address);
command.Parameters.Add(addressParameter);
var portParameter = CreatePortParameter(command, membershipEntry.SiloAddress.Endpoint.Port);
command.Parameters.Add(portParameter);
var generationParameter = CreateGenerationParameter(command, membershipEntry.SiloAddress.Generation);
command.Parameters.Add(generationParameter);
}).ConfigureAwait(continueOnCapturedContext: false);
}
public async Task<Boolean> InsertRow(MembershipEntry entry, TableVersion tableVersion)
{
try
{
//Use "0" as the eTag then Consul KV CAS will treat the operation as an insert and return false if the KV already exiats.
var consulSiloRegistration = ConsulSiloRegistrationAssembler.FromMembershipEntry(_deploymentId, entry, "0");
var insertKV = ConsulSiloRegistrationAssembler.ToKVPair(consulSiloRegistration);
var tryUpdate = await _consulClient.KV.CAS(insertKV);
if (!tryUpdate.Response)
{
_logger.Verbose("ConsulMembershipProvider failed to insert the row because a registration already exists for silo {0}.", entry.SiloAddress);
return false;
}
return true;
}
catch (Exception ex)
{
_logger.Info("ConsulMembershipProvider failed to insert registration for silo {0}; {1}.", entry.SiloAddress, ex);
throw;
}
}
internal static Tuple<MembershipEntry, String> ToMembershipEntry(ConsulSiloRegistration siloRegistration)
{
var entry = new MembershipEntry
{
SiloAddress = siloRegistration.Address,
HostName = siloRegistration.Hostname,
Status = siloRegistration.Status,
ProxyPort = siloRegistration.ProxyPort,
StartTime = siloRegistration.StartTime,
SuspectTimes = siloRegistration.SuspectingSilos.Select(silo => new Tuple<SiloAddress, DateTime>(SiloAddress.FromParsableString(silo.Id), silo.Time)).ToList(),
IAmAliveTime = siloRegistration.IAmAliveTime,
SiloName = siloRegistration.SiloName,
// Optional - only for Azure role so initialised here
RoleName = String.Empty,
UpdateZone = 0,
FaultZone = 0
};
return new Tuple<MembershipEntry, String>(entry, siloRegistration.LastIndex.ToString());
}
public async Task<Boolean> UpdateRow(MembershipEntry entry, String etag, TableVersion tableVersion)
{
//Update Silo Liveness
try
{
var siloRegistration = ConsulSiloRegistrationAssembler.FromMembershipEntry(_deploymentId, entry, etag);
var updateKV = ConsulSiloRegistrationAssembler.ToKVPair(siloRegistration);
//If the KV.CAS() call returns false then the update failed
var tryUpdate = await _consulClient.KV.CAS(updateKV);
if (!tryUpdate.Response)
{
_logger.Verbose("ConsulMembershipProvider failed the CAS check when updating the registration for silo {0}.", entry.SiloAddress);
return false;
}
return true;
}
catch (Exception ex)
{
_logger.Info("ConsulMembershipProvider failed to update the registration for silo {0}: {1}.", entry.SiloAddress, ex);
throw;
}
}
/// <summary>
///
/// </summary>
/// <param name="storage"></param>
/// <param name="query">The query to use.</param>
/// <param name="deploymentId"></param>
/// <param name="membershipEntry"></param>
/// <returns></returns>
internal static async Task UpdateIAmAliveTimeAsync(this IRelationalStorage storage, string query, string deploymentId, MembershipEntry membershipEntry)
{
await storage.ExecuteAsync(query, command =>
{
var direction = ParameterDirection.Input;
var siloIdParameter = CreateDeploymentIdParameter(command, deploymentId, direction);
command.Parameters.Add(siloIdParameter);
var iAmAliveTimeParameter = CreateIAmAliveTimeParameter(command, membershipEntry.IAmAliveTime, direction);
command.Parameters.Add(iAmAliveTimeParameter);
var addressParameter = CreateAddressParameter(command, membershipEntry.SiloAddress.Endpoint.Address, direction);
command.Parameters.Add(addressParameter);
var portParameter = CreatePortParameter(command, membershipEntry.SiloAddress.Endpoint.Port, direction);
command.Parameters.Add(portParameter);
var generationParameter = CreateGenerationParameter(command, membershipEntry.SiloAddress.Generation, direction);
command.Parameters.Add(generationParameter);
}).ConfigureAwait(continueOnCapturedContext: false);
}
public async Task UpdateIAmAlive(MembershipEntry entry)
{
var iAmAliveKV = ConsulSiloRegistrationAssembler.ToIAmAliveKVPair(_deploymentId, entry.SiloAddress, entry.IAmAliveTime);
await _consulClient.KV.Put(iAmAliveKV);
}
private static Tuple<MembershipEntry, string, int, string> CreateMembershipEntry(IDataRecord record)
{
//TODO: This is a bit of hack way to check in the current version if there's membership data or not, but if there's a start time, there's member.
DateTime? startTime = record.GetValueOrDefault<DateTime?>("StartTime");
MembershipEntry entry = null;
if(startTime.HasValue)
{
int port = record.GetValue<int>("Port");
int generation = record.GetValue<int>("Generation");
string address = record.GetValue<string>("Address");
entry = new MembershipEntry
{
SiloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Parse(address), port), generation),
HostName = record.GetValueOrDefault<string>("HostName"),
Status = record.GetValue<SiloStatus>("Status"),
ProxyPort = record.GetValueOrDefault<int>("ProxyPort"),
RoleName = record.GetValue<string>("RoleName"),
InstanceName = record.GetValue<string>("InstanceName"),
UpdateZone = record.GetValue<int>("UpdateZone"),
StartTime = startTime.GetValueOrDefault(),
FaultZone = record.GetValueOrDefault<int>("FaultZone"),
IAmAliveTime = record.GetValueOrDefault<DateTime>("IAmAliveTime")
};
//TODO: Refactor the database with regard to these.
string suspectingSilo = record.GetValueOrDefault<string>("SuspectingSilos");
string suspectingTime = record.GetValueOrDefault<string>("SuspectingTimes");
List<SiloAddress> suspectingSilos = new List<SiloAddress>();
List<DateTime> suspectingTimes = new List<DateTime>();
if(!string.IsNullOrWhiteSpace(suspectingSilo))
{
string[] silos = suspectingSilo.Split('|');
foreach(string silo in silos)
{
suspectingSilos.Add(SiloAddress.FromParsableString(silo));
}
}
if(!string.IsNullOrWhiteSpace(suspectingTime))
{
string[] times = suspectingTime.Split('|');
foreach(string time in times)
{
suspectingTimes.Add(TraceLogger.ParseDate(time));
}
}
if(suspectingSilos.Count != suspectingTimes.Count)
{
throw new OrleansException(string.Format("SuspectingSilos.Length of {0} as read from SQL table is not equal to SuspectingTimes.Length of {1}", suspectingSilos.Count, suspectingTimes.Count));
}
for(int i = 0; i < suspectingSilos.Count; ++i)
{
entry.AddSuspector(suspectingSilos[i], suspectingTimes[i]);
}
}
string etag = Convert.ToBase64String(record.GetValue<byte[]>("ETag"));
int tableVersion = (int)record.GetValueOrDefault<long>("Version");
string versionETag = Convert.ToBase64String(record.GetValueOrDefault<byte[]>("VersionETag"));
return Tuple.Create(entry, etag, tableVersion, versionETag);
}
public UpdateFaultCombo(MembershipEntry e)
{
UpdateZone = e.UpdateZone;
FaultZone = e.FaultZone;
}
Task IMembershipTable.UpdateIAmAlive(MembershipEntry entry)
{
if(logger.IsVerbose3) logger.Verbose3(string.Format("IMembershipTable.UpdateIAmAlive called with entry {0}.", entry));
if (entry == null)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.UpdateIAmAlive aborted due to null check. MembershipEntry is null.");
throw new ArgumentNullException("entry");
}
try
{
return orleansQueries.UpdateIAmAliveTimeAsync(deploymentId, entry);
}
catch(Exception ex)
{
if (logger.IsVerbose) logger.Verbose("SqlMembershipTable.UpdateIAmAlive failed: {0}", ex);
throw;
}
}
public static DateTime?HasMissedIAmAlivesSince(this MembershipEntry entry, ClusterMembershipOptions options, DateTime time)
{
var lastIAmAlive = entry.IAmAliveTime;
if (entry.IAmAliveTime.Equals(default))
