protected async Task MembershipTable_ReadAll_Insert_ReadAll()
{
MembershipTableData data = await membershipTable.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.Equal(0, data.Members.Count);
TableVersion newTableVersion = data.Version.Next();
MembershipEntry newEntry = CreateMembershipEntryForTest();
bool ok = await membershipTable.InsertRow(newEntry, newTableVersion);
Assert.True(ok, "InsertRow failed");
data = await membershipTable.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.Equal(1, data.Members.Count);
Assert.NotNull(data.Version.VersionEtag);
Assert.NotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag);
Assert.Equal(newTableVersion.Version, data.Version.Version);
var membershipEntry = data.Members[0].Item1;
string eTag = data.Members[0].Item2;
logger.Info("Membership.ReadAll returned MembershipEntry ETag={0} Entry={1}", eTag, membershipEntry);
Assert.NotNull(eTag);
Assert.NotNull(membershipEntry);
}
internal static async Task MembershipTable_ReadAll_Insert_ReadAll(IMembershipTable membership)
{
MembershipTableData data = await membership.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
TableVersion newTableVersion = data.Version.Next();
MembershipEntry newEntry = CreateMembershipEntryForTest();
bool ok = await membership.InsertRow(newEntry, newTableVersion);
Assert.IsTrue(ok, "InsertRow completed successfully");
data = await membership.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only");
Assert.IsNotNull(data.Version.VersionEtag, "New version ETag should not be null");
Assert.AreNotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag, "New VersionEtag differetnfrom last");
Assert.AreEqual(newTableVersion.Version, data.Version.Version, "New table version number");
MembershipEntry MembershipEntry = data.Members[0].Item1;
string eTag = data.Members[0].Item2;
logger.Info("Membership.ReadAll returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry);
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null");
}
public async Task <MembershipTableData> ReadRow(SiloAddress key)
{
try
{
MembershipTableData data = null;
var val = await database.StringGetAsync(clusterId);
if (!val.IsNull)
{
RedisMembershipCollection collection = serializer.Deserialize <RedisMembershipCollection>(val);
data = collection.ToMembershipTableData();
}
else
{
data = new MembershipTableData(_tableVersion);
}
return(data);
}
catch (Exception ex)
{
logger?.LogError(ex, "Redis membership table key '{0}' failed read row.", clusterId);
throw ex;
}
}
protected async Task MembershipTable_UpdateIAmAlive(bool extendedProtocol = true)
{
MembershipTableData tableData = await this.membershipTable.ReadAll();
TableVersion newTableVersion = tableData.Version.Next();
MembershipEntry newEntry = CreateMembershipEntryForTest();
bool ok = await this.membershipTable.InsertRow(newEntry, newTableVersion);
Assert.True(ok);
var amAliveTime = DateTime.UtcNow;
// This mimics the arguments MembershipOracle.OnIAmAliveUpdateInTableTimer passes in
var entry = new MembershipEntry
{
SiloAddress = newEntry.SiloAddress,
IAmAliveTime = amAliveTime
};
await this.membershipTable.UpdateIAmAlive(entry);
tableData = await this.membershipTable.ReadAll();
Tuple <MembershipEntry, string> member = tableData.Members.First();
// compare that the value is close to what we passed in, but not exactly, as the underlying store can set its own precision settings
// (ie: in SQL Server this is defined as datetime2(3), so we don't expect precision to account for less than 0.001s values)
Assert.True((amAliveTime - member.Item1.IAmAliveTime).Duration() < TimeSpan.FromMilliseconds(50), (amAliveTime - member.Item1.IAmAliveTime).Duration().ToString());
}
private MembershipTableData Convert(List<SiloInstanceRecord> entries)
{
try
{
var memEntries = new List<Tuple<MembershipEntry, string>>();
foreach (var tableEntry in entries)
{
try
{
MembershipEntry membershipEntry = Parse(tableEntry);
memEntries.Add(new Tuple<MembershipEntry, string>(membershipEntry, tableEntry.ETag.ToString()));
}
catch (Exception exc)
{
logger.Error(ErrorCode.MembershipBase,
$"Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {tableEntry}. Ignoring this entry.", exc);
}
}
var data = new MembershipTableData(memEntries, _tableVersion);
return data;
}
catch (Exception exc)
{
logger.Error(ErrorCode.MembershipBase,
$"Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {Utils.EnumerableToString(entries, e => e.ToString())}.", exc);
throw;
}
}
public async Task <MembershipTableData> ReadRow(SiloAddress siloAddress)
{
try
{
var keys = new Dictionary <string, AttributeValue>
{
{ $"{SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME}", new AttributeValue(this.clusterId) },
{ $"{SiloInstanceRecord.SILO_IDENTITY_PROPERTY_NAME}", new AttributeValue(SiloInstanceRecord.ConstructSiloIdentity(siloAddress)) }
};
var entry = await storage.ReadSingleEntryAsync(this.options.TableName, keys, fields => new SiloInstanceRecord(fields));
MembershipTableData data = entry != null?Convert(new List <SiloInstanceRecord> {
entry
}) : new MembershipTableData(this.tableVersion);
if (this.logger.IsEnabled(LogLevel.Trace))
{
this.logger.Trace("Read my entry {0} Table=" + Environment.NewLine + "{1}", siloAddress.ToLongString(), data.ToString());
}
return(data);
}
catch (Exception exc)
{
this.logger.Warn(ErrorCode.MembershipBase,
$"Intermediate error reading silo entry for key {siloAddress.ToLongString()} from the table {this.options.TableName}.", exc);
throw;
}
}
private void LogMissedIAmAlives(MembershipTableData table)
{
foreach (var pair in table.Members)
{
var entry = pair.Item1;
if (entry.SiloAddress.Equals(myAddress))
{
continue;
}
if (entry.Status != SiloStatus.Active)
{
continue;
}
var now = GetDateTimeUtcNow();
var missedSince = entry.HasMissedIAmAlivesSince(this.clusterMembershipOptions, now);
if (missedSince != null)
{
log.LogWarning(
(int)ErrorCode.MembershipMissedIAmAliveTableUpdate,
"Noticed that silo {SiloAddress} has not updated it's IAmAliveTime table column recently."
+ " Last update was at {LastUpdateTime}, now is {CurrentTime}, no update for {SinceUpdate}, which is more than {AllowedIAmAliveMissPeriod}.",
entry.SiloAddress,
missedSince,
now,
now - missedSince,
clusterMembershipOptions.AllowedIAmAliveMissPeriod);
}
}
}
private void DetectNodeMigration(MembershipTableData table, string myHostname)
{
string mySiloName = this.localSiloDetails.Name;
MembershipEntry mostRecentPreviousEntry = null;
// look for silo instances that are same as me, find most recent with Generation before me.
foreach (var entry in table.Members.Select(tuple => tuple.Item1).Where(data => mySiloName.Equals(data.SiloName)))
{
bool iAmLater = myAddress.Generation.CompareTo(entry.SiloAddress.Generation) > 0;
// more recent
if (iAmLater && (mostRecentPreviousEntry == null || entry.SiloAddress.Generation.CompareTo(mostRecentPreviousEntry.SiloAddress.Generation) > 0))
{
mostRecentPreviousEntry = entry;
}
}
if (mostRecentPreviousEntry != null)
{
bool physicalHostChanged = !myHostname.Equals(mostRecentPreviousEntry.HostName) || !myAddress.Endpoint.Equals(mostRecentPreviousEntry.SiloAddress.Endpoint);
if (physicalHostChanged)
{
string error = string.Format("Silo {0} migrated from host {1} silo address {2} to host {3} silo address {4}.",
mySiloName, myHostname, myAddress, mostRecentPreviousEntry.HostName, mostRecentPreviousEntry.SiloAddress);
log.Warn(ErrorCode.MembershipNodeMigrated, error);
}
else
{
string error = string.Format("Silo {0} restarted on same host {1} New silo address = {2} Previous silo address = {3}",
mySiloName, myHostname, myAddress, mostRecentPreviousEntry.SiloAddress);
log.Warn(ErrorCode.MembershipNodeRestarted, error);
}
}
}
private void ProcessTableUpdate(MembershipTableData table, string caller)
{
if (table is null)
{
throw new ArgumentNullException(nameof(table));
}
if (this.log.IsEnabled(LogLevel.Debug))
{
this.log.LogDebug(nameof(ProcessTableUpdate) + " (called from {Caller}) membership table {Table}", caller, table.ToString());
}
// Update the current membership snapshot.
var(localSiloEntry, _) = this.GetOrCreateLocalSiloEntry(table, this.CurrentStatus);
var updated = MembershipTableSnapshot.Create(localSiloEntry, table);
if (this.updates.TryPublish(updated))
{
this.LogMissedIAmAlives(table);
this.log.LogInformation(
(int)ErrorCode.MembershipReadAll_2,
nameof(ProcessTableUpdate) + " (called from {Caller}) membership table: {Table}",
caller,
table.WithoutDuplicateDeads().ToString());
}
}
private void LogMissedIAmAlives(MembershipTableData table)
{
foreach (var pair in table.Members)
{
var entry = pair.Item1;
if (entry.SiloAddress.Equals(myAddress))
{
continue;
}
if (entry.Status != SiloStatus.Active)
{
continue;
}
var now = DateTime.UtcNow;
var missedSince = entry.HasMissedIAmAlivesSince(this.clusterMembershipOptions, now);
if (missedSince != null)
{
log.Warn(
ErrorCode.MembershipMissedIAmAliveTableUpdate,
$"Noticed that silo {entry.SiloAddress} has not updated it's IAmAliveTime table column recently."
+ $" Last update was at {missedSince}, now is {now}, no update for {now - missedSince}, which is more than {this.clusterMembershipOptions.AllowedIAmAliveMissPeriod}.");
}
}
}
public async Task <MembershipTableData> ReadAll()
{
var entries = new List <Tuple <MembershipEntry, string> >();
TableVersion version = await ReadTableVersion();
try
{
var recordSet = _client.Query(null, new Statement()
{
Filter = Filter.Equal("clusterid", _clusterOptions.ClusterId),
Namespace = _options.Namespace,
SetName = _options.SetName
});
while (recordSet.Next())
{
entries.Add(
new Tuple <MembershipEntry, string>(
ParseMembershipEntryRecord(recordSet.Record),
recordSet.Record.generation.ToString()));
}
}
catch (Exception exp)
{
}
var data = new MembershipTableData(entries, version);
return(data);
}
public static MembershipTableSnapshot Create(MembershipEntry localSiloEntry, MembershipTableData table)
{
if (table is null)
{
throw new ArgumentNullException(nameof(table));
}
var entries = ImmutableDictionary.CreateBuilder <SiloAddress, MembershipEntry>();
if (table.Members != null)
{
foreach (var item in table.Members)
{
var entry = item.Item1;
entries.Add(entry.SiloAddress, entry);
}
}
if (entries.TryGetValue(localSiloEntry.SiloAddress, out var existing))
{
entries[localSiloEntry.SiloAddress] = existing.WithStatus(localSiloEntry.Status);
}
else
{
entries[localSiloEntry.SiloAddress] = localSiloEntry;
}
var version = new MembershipVersion(table.Version.Version);
return(new MembershipTableSnapshot(version, entries.ToImmutable()));
}
protected async Task MembershipTable_UpdateIAmAlive(bool extendedProtocol = true)
{
MembershipTableData tableData = await membershipTable.ReadAll();
TableVersion newTableVersion = tableData.Version.Next();
MembershipEntry newEntry = CreateMembershipEntryForTest();
bool ok = await membershipTable.InsertRow(newEntry, newTableVersion);
Assert.True(ok);
var amAliveTime = DateTime.UtcNow;
// This mimics the arguments MembershipOracle.OnIAmAliveUpdateInTableTimer passes in
var entry = new MembershipEntry
{
SiloAddress = newEntry.SiloAddress,
IAmAliveTime = amAliveTime
};
await membershipTable.UpdateIAmAlive(entry);
tableData = await membershipTable.ReadAll();
Tuple <MembershipEntry, string> member = tableData.Members.First();
Assert.True((amAliveTime - member.Item1.IAmAliveTime).Duration() < TimeSpan.FromMilliseconds(1));
}
public async Task <MembershipTableData> ReadRow(SiloAddress key)
{
var name = ConstructSiloEntityId(key);
try
{
var versionEntity = await this.GetClusterVersion();
SiloEntity entity = default;
try
{
entity = ((JObject)await this._kubeClient.GetNamespacedCustomObjectAsync(
Constants.ORLEANS_GROUP,
Constants.PROVIDER_MODEL_VERSION,
this._namespace,
SiloEntity.PLURAL,
name
))?.ToObject <SiloEntity>();
}
catch (HttpOperationException ex)
{
if (ex.Response.StatusCode != HttpStatusCode.NotFound)
{
throw;
}
}
var version = default(TableVersion);
if (versionEntity != null)
{
version = new TableVersion(versionEntity.ClusterVersion, versionEntity.Metadata.ResourceVersion);
}
else
{
this._logger?.LogError("Initial ClusterVersionEntity entity doesn't exist.");
}
var memEntries = new List <Tuple <MembershipEntry, string> >();
if (entity != null)
{
var membershipEntry = ParseEntity(entity);
memEntries.Add(new Tuple <MembershipEntry, string>(membershipEntry, entity.Metadata.ResourceVersion));
}
var data = new MembershipTableData(memEntries, version);
return(data);
}
catch (Exception exc)
{
this._logger?.LogError(exc, "Failure reading silo entry {name} for cluster id {clusterId}.", name, this._clusterOptions.ClusterId);
throw;
}
}
public async Task GatewaySelection_SqlServer()
{
string testName = Guid.NewGuid().ToString();// TestContext.TestName;
Guid serviceId = Guid.NewGuid();
GlobalConfiguration cfg = new GlobalConfiguration
{
ServiceId = serviceId,
DeploymentId = testName,
DataConnectionString = TestHelper.TestUtils.GetSqlConnectionString()
};
var membership = new SqlMembershipTable();
var logger = LogManager.GetLogger(membership.GetType().Name);
await membership.InitializeMembershipTable(cfg, true, logger);
IMembershipTable membershipTable = membership;
// Pre-populate gateway table with data
int count = 1;
foreach (Uri gateway in gatewayAddressUris)
{
output.WriteLine("Adding gataway data for {0}", gateway);
SiloAddress siloAddress = gateway.ToSiloAddress();
Assert.IsNotNull(siloAddress, "Unable to get SiloAddress from Uri {0}", gateway);
MembershipEntry MembershipEntry = new MembershipEntry
{
SiloAddress = siloAddress,
HostName = gateway.Host,
Status = SiloStatus.Active,
ProxyPort = gateway.Port,
StartTime = DateTime.UtcNow
};
var tableVersion = new TableVersion(count, Guid.NewGuid().ToString());
output.WriteLine("Inserting gataway data for {0} with TableVersion={1}", MembershipEntry, tableVersion);
bool ok = await membershipTable.InsertRow(MembershipEntry, tableVersion);
count++;
Assert.IsTrue(ok, "Membership record should have been written OK but were not: {0}", MembershipEntry);
output.WriteLine("Successfully inserted Membership row {0}", MembershipEntry);
}
MembershipTableData data = await membershipTable.ReadAll();
Assert.IsNotNull(data, "MembershipTableData returned");
Assert.AreEqual(gatewayAddressUris.Count, data.Members.Count, "Number of gateway records read");
IGatewayListProvider listProvider = membership;
Test_GatewaySelection(listProvider);
}
protected async Task MembershipTable_ReadRow_Insert_Read()
{
MembershipTableData data = await membershipTable.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.Equal(0, data.Members.Count);
TableVersion newTableVersion = data.Version.Next();
MembershipEntry newEntry = CreateMembershipEntryForTest();
bool ok = await membershipTable.InsertRow(newEntry, newTableVersion);
Assert.True(ok, "InsertRow failed");
ok = await membershipTable.InsertRow(newEntry, newTableVersion);
Assert.False(ok, "InsertRow should have failed - same entry, old table version");
ok = await membershipTable.InsertRow(CreateMembershipEntryForTest(), newTableVersion);
Assert.False(ok, "InsertRow should have failed - new entry, old table version");
data = await membershipTable.ReadAll();
Assert.Equal(1, data.Version.Version);
var nextTableVersion = data.Version.Next();
ok = await membershipTable.InsertRow(newEntry, nextTableVersion);
Assert.False(ok, "InsertRow should have failed - duplicate entry");
data = await membershipTable.ReadAll();
Assert.Equal(1, data.Members.Count);
data = await membershipTable.ReadRow(newEntry.SiloAddress);
Assert.Equal(newTableVersion.Version, data.Version.Version);
logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", data.Version, data);
Assert.Equal(1, data.Members.Count);
Assert.NotNull(data.Version.VersionEtag);
Assert.NotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag);
Assert.Equal(newTableVersion.Version, data.Version.Version);
var membershipEntry = data.Members[0].Item1;
string eTag = data.Members[0].Item2;
logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} Entry={1}", eTag, membershipEntry);
Assert.NotNull(eTag);
Assert.NotNull(membershipEntry);
}
// read the table
// find all currently active nodes and test pings to all of them
// try to ping all
// if all pings succeeded
// try to change my status to Active and in the same write transaction update Membership version row, conditioned on both etags
// if failed (on ping or on write exception or on etag) - retry the whole AttemptToJoinActiveNodes
private async Task <bool> TryUpdateMyStatusGlobalOnce(SiloStatus newStatus)
{
var table = await membershipTableProvider.ReadAll();
if (log.IsEnabled(LogLevel.Debug))
{
log.LogDebug(
"TryUpdateMyStatusGlobalOnce: Read{Selection} Membership table {Table}",
(newStatus.Equals(SiloStatus.Active) ? "All" : " my entry from"),
table.ToString());
}
LogMissedIAmAlives(table);
var(myEntry, myEtag) = this.GetOrCreateLocalSiloEntry(table, newStatus);
if (myEntry.Status == SiloStatus.Dead && myEntry.Status != newStatus)
{
this.log.LogWarning(
(int)ErrorCode.MembershipFoundMyselfDead1,
"I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {Entry}.",
myEntry.ToFullString(full: true));
this.KillMyselfLocally($"I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {(myEntry.ToFullString(full: true))}.");
return(true);
}
var now = GetDateTimeUtcNow();
if (newStatus == SiloStatus.Dead)
{
myEntry.AddSuspector(myAddress, now); // add the killer (myself) to the suspect list, for easier diagnostics later on.
}
myEntry.Status = newStatus;
myEntry.IAmAliveTime = now;
bool ok;
TableVersion next = table.Version.Next();
if (myEtag != null) // no previous etag for my entry -> its the first write to this entry, so insert instead of update.
{
ok = await membershipTableProvider.UpdateRow(myEntry, myEtag, next);
}
else
{
ok = await membershipTableProvider.InsertRow(myEntry, next);
}
if (ok)
{
this.CurrentStatus = newStatus;
var entries = table.Members.ToDictionary(e => e.Item1.SiloAddress, e => e);
entries[myEntry.SiloAddress] = Tuple.Create(myEntry, myEtag);
var updatedTable = new MembershipTableData(entries.Values.ToList(), next);
this.ProcessTableUpdate(updatedTable, nameof(TryUpdateMyStatusGlobalOnce));
}
return(ok);
}
public async Task <MembershipTableData> ReadRow(SiloAddress key)
{
var id = ConstructSiloEntityId(key);
try
{
var query = this._container
.GetItemQueryIterator <dynamic>(
new QueryDefinition(READ_ENTRY_QUERY).WithParameter("@siloId", id),
requestOptions: new QueryRequestOptions
{
PartitionKey = new PartitionKey(this._clusterOptions.ClusterId)
}
);
var docs = await query.ReadNextAsync();
var versionDoc = docs.Where(i => i.EntityType == nameof(ClusterVersionEntity)).SingleOrDefault();
ClusterVersionEntity clusterVersionEntity = versionDoc != null?ClusterVersionEntity.FromDocument(versionDoc) : default;
var siloEntities = docs.Where(i => i.EntityType == nameof(SiloEntity)).Select <dynamic, SiloEntity>(d => SiloEntity.FromDocument(d));
TableVersion version = null;
if (clusterVersionEntity != null)
{
version = new TableVersion(clusterVersionEntity.ClusterVersion, clusterVersionEntity.ETag);
}
else
{
this._logger.LogError("Initial ClusterVersionEntity entity doesn't exist.");
}
var memEntries = new List <Tuple <MembershipEntry, string> >();
foreach (var entity in siloEntities)
{
try
{
MembershipEntry membershipEntry = ParseEntity(entity);
memEntries.Add(new Tuple <MembershipEntry, string>(membershipEntry, entity.ETag));
}
catch (Exception exc)
{
this._logger.LogError(exc, "Failure reading membership row.");
throw;
}
}
var data = new MembershipTableData(memEntries, version);
return(data);
}
catch (Exception exc)
{
this._logger.LogWarning($"Failure reading silo entry {id} for cluster id {this._clusterOptions.ClusterId}: {exc}");
throw;
}
}
private (MembershipEntry Entry, string ETag) GetOrCreateLocalSiloEntry(MembershipTableData table, SiloStatus currentStatus)
{
if (table.Contains(this.myAddress))
{
var myTuple = table.Get(this.myAddress);
return(myTuple.Item1.Copy(), myTuple.Item2);
}
return(this.CreateLocalSiloEntry(currentStatus), null);
}
protected async Task MembershipTable_CleanupDefunctSiloEntries(bool extendedProtocol = true)
{
MembershipTableData data = await membershipTable.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.Equal(0, data.Members.Count);
TableVersion newTableVersion = data.Version.Next();
var oldEntryDead = CreateMembershipEntryForTest();
oldEntryDead.IAmAliveTime = oldEntryDead.IAmAliveTime.AddDays(-10);
oldEntryDead.StartTime = oldEntryDead.StartTime.AddDays(-10);
oldEntryDead.Status = SiloStatus.Dead;
bool ok = await membershipTable.InsertRow(oldEntryDead, newTableVersion);
var table = await membershipTable.ReadAll();
Assert.True(ok, "InsertRow Dead failed");
newTableVersion = table.Version.Next();
var oldEntryJoining = CreateMembershipEntryForTest();
oldEntryJoining.IAmAliveTime = oldEntryJoining.IAmAliveTime.AddDays(-10);
oldEntryJoining.StartTime = oldEntryJoining.StartTime.AddDays(-10);
oldEntryJoining.Status = SiloStatus.Joining;
ok = await membershipTable.InsertRow(oldEntryJoining, newTableVersion);
table = await membershipTable.ReadAll();
Assert.True(ok, "InsertRow Joining failed");
newTableVersion = table.Version.Next();
var newEntry = CreateMembershipEntryForTest();
ok = await membershipTable.InsertRow(newEntry, newTableVersion);
Assert.True(ok, "InsertRow failed");
data = await membershipTable.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.Equal(3, data.Members.Count);
await membershipTable.CleanupDefunctSiloEntries(oldEntryDead.IAmAliveTime.AddDays(3));
data = await membershipTable.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.Equal(1, data.Members.Count);
}
protected async Task MembershipTable_ReadRow_Insert_Read()
{
MembershipTableData data = await membershipTable.ReadAll();
//TableVersion tableVersion = data.Version;
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
TableVersion newTableVersion = data.Version.Next();
MembershipEntry newEntry = CreateMembershipEntryForTest();
bool ok = await membershipTable.InsertRow(newEntry, newTableVersion);
Assert.IsTrue(ok, "InsertRow failed");
ok = await membershipTable.InsertRow(newEntry, newTableVersion);
Assert.IsFalse(ok, "InsertRow should have failed - same entry, old table version");
ok = await membershipTable.InsertRow(CreateMembershipEntryForTest(), newTableVersion);
Assert.IsFalse(ok, "InsertRow should have failed - new entry, old table version");
data = await membershipTable.ReadAll();
var nextTableVersion = data.Version.Next();
ok = await membershipTable.InsertRow(newEntry, nextTableVersion);
Assert.IsFalse(ok, "InsertRow should have failed - duplicate entry");
data = await membershipTable.ReadAll();
Assert.AreEqual(1, data.Members.Count, "only one row should have been inserted");
data = await membershipTable.ReadRow(newEntry.SiloAddress);
Assert.AreEqual(newTableVersion.Version, data.Version.Version);
logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", data.Version, data);
Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only");
Assert.IsNotNull(data.Version.VersionEtag, "New version ETag should not be null");
Assert.AreNotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag, "New VersionEtag differetnfrom last");
Assert.AreEqual(newTableVersion.Version, data.Version.Version);
MembershipEntry MembershipEntry = data.Members[0].Item1;
string eTag = data.Members[0].Item2;
logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry);
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null");
}
public async Task <MembershipTableData> ReadAll()
{
try
{
var versionEntity = await this.GetClusterVersion();
var entryEntities = await this.GetSilos();
var version = default(TableVersion);
if (versionEntity != null)
{
version = new TableVersion(versionEntity.ClusterVersion, versionEntity.Metadata.ResourceVersion);
}
else
{
this._logger?.LogError("Initial ClusterVersionEntity entity doesn't exist.");
}
var memEntries = new List <Tuple <MembershipEntry, string> >();
foreach (var entity in entryEntities)
{
try
{
MembershipEntry membershipEntry = ParseEntity(entity);
memEntries.Add(new Tuple <MembershipEntry, string>(membershipEntry, entity.Metadata.ResourceVersion));
}
catch (Exception exc)
{
this._logger?.LogWarning(exc, "Failure reading all membership records from Kubernetes");
throw;
}
}
var data = new MembershipTableData(memEntries, version);
return(data);
}
catch (Exception exc)
{
this._logger?.LogWarning(exc, "Failure reading all silo entries for cluster id {clusterId}", this._clusterOptions.ClusterId);
throw;
}
}
public async Task <MembershipTableData> ReadAll()
{
try
{
//first read just the version row so that we can check for version consistency
var versionEntryKeys = new Dictionary <string, AttributeValue>
{
{ $"{SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME}", new AttributeValue(this.clusterId) },
{ $"{SiloInstanceRecord.SILO_IDENTITY_PROPERTY_NAME}", new AttributeValue(SiloInstanceRecord.TABLE_VERSION_ROW) }
};
var versionRow = await this.storage.ReadSingleEntryAsync(this.options.TableName, versionEntryKeys,
fields => new SiloInstanceRecord(fields));
if (versionRow == null)
{
throw new KeyNotFoundException("No version row found for membership table");
}
var keys = new Dictionary <string, AttributeValue> {
{ $":{SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME}", new AttributeValue(this.clusterId) }
};
var records = await this.storage.QueryAllAsync(this.options.TableName, keys, $"{SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME} = :{SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME}", item => new SiloInstanceRecord(item));
if (records.Any(record => record.MembershipVersion > versionRow.MembershipVersion))
{
this.logger.LogWarning((int)ErrorCode.MembershipBase, "Found an inconsistency while reading all silo entries");
//not expecting this to hit often, but if it does, should put in a limit
return(await this.ReadAll());
}
MembershipTableData data = Convert(records);
if (this.logger.IsEnabled(LogLevel.Trace))
{
this.logger.LogTrace("ReadAll Table {Table}", data.ToString());
}
return(data);
}
catch (Exception exc)
{
this.logger.LogWarning(
(int)ErrorCode.MembershipBase,
exc,
"Intermediate error reading all silo entries {TableName}.",
options.TableName);
throw;
}
}
public async Task <MembershipTableData> ReadRow(SiloAddress key)
{
var id = ConstructSiloEntityId(key);
try
{
var spResponse = await this._dbClient.ExecuteStoredProcedureAsync <ReadResponse>(
UriFactory.CreateStoredProcedureUri(this._options.DB, this._options.Collection, READ_SILO_SPROC),
new RequestOptions { PartitionKey = new PartitionKey(this._options.ClusterId) },
this._options.ClusterId, id);
ClusterVersionEntity versionEntity = spResponse.Response.ClusterVersion;
List <SiloEntity> entryEntities = spResponse.Response.Silos;
TableVersion version = null;
if (versionEntity != null)
{
version = new TableVersion(versionEntity.ClusterVersion, versionEntity.ETag);
}
else
{
this._logger.LogError("Initial ClusterVersionEntity entity doesn't exist.");
}
var memEntries = new List <Tuple <MembershipEntry, string> >();
foreach (var entity in entryEntities)
{
try
{
MembershipEntry membershipEntry = ParseEntity(entity);
memEntries.Add(new Tuple <MembershipEntry, string>(membershipEntry, entity.ETag));
}
catch (Exception exc)
{
//TODO: Log it
throw;
}
}
var data = new MembershipTableData(memEntries, version);
return(data);
}
catch (Exception exc)
{
this._logger.LogWarning($"Failure reading silo entry {id} for cluster id {this._options.ClusterId}: {exc}");
throw;
}
}
public async Task <MembershipTableData> ReadRow(SiloAddress key)
{
var siloId = GetSiloEntityId(key);
var record = await _client.Get(null, Task.Factory.CancellationToken, new Key(_options.Namespace, _options.SetName, siloId));
var entries = new List <Tuple <MembershipEntry, string> >();
entries.Add(new Tuple <MembershipEntry, string>(ParseMembershipEntryRecord(record), record.generation.ToString()));
TableVersion version = await ReadTableVersion();
var data = new MembershipTableData(entries, version);
return(data);
}
public async Task AzureMembership_ReadAll_0()
{
MembershipTableData data = await membership.ReadAll();
TableVersion tableVersion = data.Version;
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
string eTag = tableVersion.VersionEtag;
int ver = tableVersion.Version;
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.AreEqual(0, ver, "Initial tabel version should be zero");
}
private async Task MembershipTable_InsertRow(IMembershipTable membership)
{
MembershipEntry MembershipEntry = CreateMembershipEntryForTest();
MembershipTableData MembershipData = await membership.ReadAll();
Assert.IsNotNull(MembershipData, "Membership Data not null");
Assert.AreEqual(0, MembershipData.Members.Count, "Should be no data initially: {0}", MembershipData);
bool ok = await membership.InsertRow(MembershipEntry, MembershipData.Version);
Assert.IsTrue(ok, "InsertRow OK");
MembershipData = await membership.ReadAll();
Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after insert: {0}", MembershipData);
}
/// <summary>
/// Maps the specified source.
/// </summary>
/// <param name="src">The source.</param>
/// <param name="dst">The DST.</param>
/// <returns></returns>
internal static MembershipTableData Map(
List <OrleansEFMembership> src,
MembershipTableData dst = null
)
{
var entries = src.Select(a =>
{
return(new Tuple <MembershipEntry, string>(
new MembershipEntry
{
SiloAddress = Map(
a.Address,
(int)a.Port,
a.Generation
),
Status = (SiloStatus)a.Status,
ProxyPort = (int)a.ProxyPort,
HostName = a.HostName,
SiloName = a.SiloName,
StartTime = a.StartTime,
IAmAliveTime = a.IAmAliveTime,
UpdateZone = a.UpdateZone,
FaultZone = a.FaultZone,
SuspectTimes = a.SuspectTimes?.Split(";").Where(b =>
!string.IsNullOrWhiteSpace(b)
).Select(b =>
{
var split = b.Split("::");
return new Tuple <SiloAddress, DateTime>(
SiloAddress.FromParsableString(split[0]),
DateTimeOffset.FromUnixTimeMilliseconds(long.Parse(split[1])).UtcDateTime
);
}).ToList(),
},
a.DeploymentId
));
}).ToList();
dst = new MembershipTableData(
entries,
new TableVersion(1, "whatever")
);
return(dst);
}
// Only used with MembershipTableGrain to wiat for primary to start.
internal async Task WaitForTableToInit(IMembershipTable membershipTable)
{
var timespan = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(5);
// This is a quick temporary solution to enable primary node to start fully before secondaries.
// Secondary silos waits untill GrainBasedMembershipTable is created.
for (int i = 0; i < 100; i++)
{
bool needToWait = false;
try
{
MembershipTableData table = await membershipTable.ReadAll().WithTimeout(timespan);
if (table.Members.Any(tuple => tuple.Item1.IsPrimary))
{
logger.Info(ErrorCode.MembershipTableGrainInit1,
"-Connected to membership table provider and also found primary silo registered in the table.");
return;
}
logger.Info(ErrorCode.MembershipTableGrainInit2,
"-Connected to membership table provider but did not find primary silo registered in the table. Going to try again for a {0}th time.", i);
}
catch (Exception exc)
{
var type = exc.GetBaseException().GetType();
if (type == typeof(TimeoutException) || type == typeof(OrleansException))
{
logger.Info(ErrorCode.MembershipTableGrainInit3,
"-Waiting for membership table provider to initialize. Going to sleep for {0} and re-try to reconnect.", timespan);
needToWait = true;
}
else
{
logger.Info(ErrorCode.MembershipTableGrainInit4, "-Membership table provider failed to initialize. Giving up.");
throw;
}
}
if (needToWait)
{
await Task.Delay(timespan);
}
}
}
internal static async Task MembershipTable_ReadRow_EmptyTable(IMembershipTable membership, SiloAddress siloAddress)
{
MembershipTableData data = await membership.ReadRow(siloAddress);
TableVersion tableVersion = data.Version;
logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", tableVersion, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
string eTag = tableVersion.VersionEtag;
int ver = tableVersion.Version;
logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} TableVersion={1}", eTag, tableVersion);
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.AreEqual(0, ver, "Initial table version should be zero");
}
private MembershipTableData Convert(List<Tuple<SiloInstanceTableEntry, string>> entries)
{
try
{
var memEntries = new List<Tuple<MembershipEntry, string>>();
TableVersion tableVersion = null;
foreach (var tuple in entries)
{
var tableEntry = tuple.Item1;
if (tableEntry.RowKey.Equals(SiloInstanceTableEntry.TABLE_VERSION_ROW))
{
tableVersion = new TableVersion(Int32.Parse(tableEntry.MembershipVersion), tuple.Item2);
}
else
{
try
{
MembershipEntry membershipEntry = Parse(tableEntry);
memEntries.Add(new Tuple<MembershipEntry, string>(membershipEntry, tuple.Item2));
}
catch (Exception exc)
{
logger.Error(ErrorCode.AzureTable_61, String.Format(
"Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {0}. Ignoring this entry.",
tableEntry), exc);
}
}
}
var data = new MembershipTableData(memEntries, tableVersion);
return data;
}
catch (Exception exc)
{
logger.Error(ErrorCode.AzureTable_60, String.Format(
"Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {0}.",
Utils.EnumerableToString(entries, tuple => tuple.Item1.ToString())), exc);
throw;
}
}
private MembershipTableData Convert(List<SiloInstanceRecord> entries)
{
try
{
var memEntries = new List<Tuple<MembershipEntry, string>>();
foreach (var tableEntry in entries)
{
try
{
MembershipEntry membershipEntry = Parse(tableEntry);
memEntries.Add(new Tuple<MembershipEntry, string>(membershipEntry, tableEntry.ETag.ToString()));
}
catch (Exception exc)
{
logger.Error(ErrorCode.MembershipBase,
$"Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {tableEntry}. Ignoring this entry.", exc);
}
}
var data = new MembershipTableData(memEntries, _tableVersion);
return data;
}
catch (Exception exc)
{
logger.Error(ErrorCode.AzureTable_60,
$"Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {Utils.EnumerableToString(entries, e => e.ToString())}.", exc);
throw;
}
}
