internal static async Task MembershipTable_ReadAll_Insert_ReadAll(IMembershipTable membership)
{
MembershipTableData data = await membership.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
TableVersion newTableVersion = data.Version.Next();
MembershipEntry newEntry = CreateMembershipEntryForTest();
bool ok = await membership.InsertRow(newEntry, newTableVersion);
Assert.IsTrue(ok, "InsertRow completed successfully");
data = await membership.ReadAll();
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only");
Assert.IsNotNull(data.Version.VersionEtag, "New version ETag should not be null");
Assert.AreNotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag, "New VersionEtag differetnfrom last");
Assert.AreEqual(newTableVersion.Version, data.Version.Version, "New table version number");
MembershipEntry MembershipEntry = data.Members[0].Item1;
string eTag = data.Members[0].Item2;
logger.Info("Membership.ReadAll returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry);
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null");
}
// read the table
// find all currently active nodes and test pings to all of them
// try to ping all
// if all pings succeeded
// try to change my status to Active and in the same write transaction update Membership version row, conditioned on both etags
// if failed (on ping or on write exception or on etag) - retry the whole AttemptToJoinActiveNodes
private async Task <bool> TryUpdateMyStatusGlobalOnce(SiloStatus newStatus)
{
var table = await membershipTableProvider.ReadAll();
if (log.IsEnabled(LogLevel.Debug))
{
log.LogDebug(
"TryUpdateMyStatusGlobalOnce: Read{Selection} Membership table {Table}",
(newStatus.Equals(SiloStatus.Active) ? "All" : " my entry from"),
table.ToString());
}
LogMissedIAmAlives(table);
var(myEntry, myEtag) = this.GetOrCreateLocalSiloEntry(table, newStatus);
if (myEntry.Status == SiloStatus.Dead && myEntry.Status != newStatus)
{
this.log.LogWarning(
(int)ErrorCode.MembershipFoundMyselfDead1,
"I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {Entry}.",
myEntry.ToFullString(full: true));
this.KillMyselfLocally($"I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {(myEntry.ToFullString(full: true))}.");
return(true);
}
var now = GetDateTimeUtcNow();
if (newStatus == SiloStatus.Dead)
{
myEntry.AddSuspector(myAddress, now); // add the killer (myself) to the suspect list, for easier diagnostics later on.
}
myEntry.Status = newStatus;
myEntry.IAmAliveTime = now;
bool ok;
TableVersion next = table.Version.Next();
if (myEtag != null) // no previous etag for my entry -> its the first write to this entry, so insert instead of update.
{
ok = await membershipTableProvider.UpdateRow(myEntry, myEtag, next);
}
else
{
ok = await membershipTableProvider.InsertRow(myEntry, next);
}
if (ok)
{
this.CurrentStatus = newStatus;
var entries = table.Members.ToDictionary(e => e.Item1.SiloAddress, e => e);
entries[myEntry.SiloAddress] = Tuple.Create(myEntry, myEtag);
var updatedTable = new MembershipTableData(entries.Values.ToList(), next);
this.ProcessTableUpdate(updatedTable, nameof(TryUpdateMyStatusGlobalOnce));
}
return(ok);
}
internal static async Task MembershipTable_InsertRow(IMembershipTable membership)
{
var membershipEntry = CreateMembershipEntryForTest();
var data = await membership.ReadAll();
Assert.IsNotNull(data, "Membership Data not null");
Assert.AreEqual(0, data.Members.Count, "Should be no data initially: {0}", data);
bool ok = await membership.InsertRow(membershipEntry, data.Version.Next());
Assert.IsTrue(ok, "InsertRow OK");
data = await membership.ReadAll();
Assert.AreEqual(1, data.Members.Count, "Should be one row after insert: {0}", data);
}
internal static async Task MembershipTable_ReadRow_Insert_Read(IMembershipTable membership)
{
MembershipTableData data = await membership.ReadAll();
//TableVersion tableVersion = data.Version;
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
TableVersion newTableVersion = data.Version.Next();
MembershipEntry newEntry = CreateMembershipEntryForTest();
bool ok = await membership.InsertRow(newEntry, newTableVersion);
Assert.IsTrue(ok, "InsertRow completed successfully");
data = await membership.ReadRow(newEntry.SiloAddress);
logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", data.Version, data);
Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only");
Assert.IsNotNull(data.Version.VersionEtag, "New version ETag should not be null");
Assert.AreNotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag, "New VersionEtag differetnfrom last");
Assert.AreEqual(newTableVersion.Version, data.Version.Version, "New table version number");
MembershipEntry MembershipEntry = data.Members[0].Item1;
string eTag = data.Members[0].Item2;
logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry);
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null");
}
internal static async Task MembershipTable_InsertRow(IMembershipTable membership)
{
var membershipEntry = CreateMembershipEntryForTest();
var data = await membership.ReadAll();
Assert.IsNotNull(data, "Membership Data not null");
Assert.AreEqual(0, data.Members.Count, "Should be no data initially: {0}", data);
bool ok = await membership.InsertRow(membershipEntry, data.Version.Next());
Assert.IsTrue(ok, "InsertRow OK");
data = await membership.ReadAll();
Assert.AreEqual(1, data.Members.Count, "Should be one row after insert: {0}", data);
}
private async Task MembershipTable_InsertRow(IMembershipTable membership)
{
MembershipEntry MembershipEntry = CreateMembershipEntryForTest();
MembershipTableData MembershipData = await membership.ReadAll();
Assert.IsNotNull(MembershipData, "Membership Data not null");
Assert.AreEqual(0, MembershipData.Members.Count, "Should be no data initially: {0}", MembershipData);
bool ok = await membership.InsertRow(MembershipEntry, MembershipData.Version);
Assert.IsTrue(ok, "InsertRow OK");
MembershipData = await membership.ReadAll();
Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after insert: {0}", MembershipData);
}
internal static async Task MembershipTable_UpdateRow(IMembershipTable membership)
{
MembershipEntry data = CreateMembershipEntryForTest();
MembershipTableData tableData = await membership.ReadAll();
//TableVersion tableVer = tableData.Version;
Assert.IsNotNull(tableData.Version, "TableVersion should not be null");
Assert.AreEqual(0, tableData.Version.Version, "TableVersion should be zero");
Assert.AreEqual(0, tableData.Members.Count, "Should be no data initially: {0}", tableData);
TableVersion newTableVer = tableData.Version.Next();
logger.Info("Calling InsertRow with Entry = {0} TableVersion = {1}", data, newTableVer);
bool ok = await membership.InsertRow(data, newTableVer);
Assert.IsTrue(ok, "InsertRow OK");
tableData = await membership.ReadAll();
Assert.IsNotNull(tableData.Version, "TableVersion should not be null");
Assert.AreEqual(1, tableData.Version.Version, "TableVersion should be 1");
Assert.AreEqual(1, tableData.Members.Count, "Should be one row after insert: {0}", tableData);
Tuple <MembershipEntry, string> insertedData = tableData.Get(data.SiloAddress);
Assert.IsNotNull(insertedData.Item2, "ETag should not be null");
insertedData.Item1.Status = SiloStatus.Active;
newTableVer = tableData.Version.Next();
logger.Info("Calling UpdateRow with Entry = {0} eTag = {1} New TableVersion={2}", insertedData.Item1, insertedData.Item2, newTableVer);
ok = await membership.UpdateRow(insertedData.Item1, insertedData.Item2, newTableVer);
tableData = await membership.ReadAll();
Assert.IsNotNull(tableData.Version, "TableVersion should not be null");
Assert.AreEqual(2, tableData.Version.Version, "TableVersion should be 2");
Assert.AreEqual(1, tableData.Members.Count, "Should be one row after insert: {0}", tableData);
Assert.IsTrue(ok, "UpdateRow OK - Table Data = {0}", tableData);
}
internal static async Task MembershipTable_ReadAll_Insert_ReadAll(IMembershipTable membership, SiloAddress siloAddress)
{
MembershipTableData data = await membership.ReadAll();
TableVersion tableVersion = data.Version;
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
DateTime now = DateTime.UtcNow;
MembershipEntry entry = new MembershipEntry
{
SiloAddress = siloAddress,
StartTime = now,
Status = SiloStatus.Active,
};
TableVersion newTableVersion = tableVersion.Next();
bool ok = await membership.InsertRow(entry, newTableVersion);
Assert.IsTrue(ok, "InsertRow completed successfully");
data = await membership.ReadAll();
tableVersion = data.Version;
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, data);
Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only");
Assert.IsNotNull(tableVersion.VersionEtag, "New version ETag should not be null");
Assert.AreNotEqual(newTableVersion.VersionEtag, tableVersion.VersionEtag, "New VersionEtag differetnfrom last");
Assert.AreEqual(newTableVersion.Version, tableVersion.Version, "New table version number");
MembershipEntry MembershipEntry = data.Members[0].Item1;
string eTag = data.Members[0].Item2;
logger.Info("Membership.ReadAll returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry);
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null");
}
internal static async Task MembershipTable_ReadAll_EmptyTable(IMembershipTable membership)
{
var data = await membership.ReadAll();
Assert.IsNotNull(data, "Membership Data not null");
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
Assert.IsNotNull(data.Version.VersionEtag, "ETag should not be null");
Assert.AreEqual(0, data.Version.Version, "Initial tabel version should be zero");
}
internal static async Task MembershipTable_ReadAll_EmptyTable(IMembershipTable membership)
{
var data = await membership.ReadAll();
Assert.IsNotNull(data, "Membership Data not null");
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
Assert.IsNotNull(data.Version.VersionEtag, "ETag should not be null");
Assert.AreEqual(0, data.Version.Version, "Initial tabel version should be zero");
}
private async Task MembershipTable_UpdateRow(IMembershipTable membership)
{
MembershipEntry MembershipEntry = CreateMembershipEntryForTest();
MembershipTableData MembershipData = await membership.ReadAll();
TableVersion tableVer = MembershipData.Version;
Assert.AreEqual(0, MembershipData.Members.Count, "Should be no data initially: {0}", MembershipData);
logger.Info("Calling InsertRow with Entry = {0} TableVersion = {1}", MembershipEntry, tableVer);
bool ok = await membership.InsertRow(MembershipEntry, tableVer);
Assert.IsTrue(ok, "InsertRow OK");
MembershipData = await membership.ReadAll();
Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after insert: {0}", MembershipData);
Tuple <MembershipEntry, string> newEntryData = MembershipData.Get(MembershipEntry.SiloAddress);
string eTag = newEntryData.Item2;
Assert.IsNotNull(eTag, "ETag should not be null");
tableVer = MembershipData.Version;
Assert.IsNotNull(tableVer, "TableVersion should not be null");
tableVer = tableVer.Next();
MembershipEntry = CreateMembershipEntryForTest();
MembershipEntry.Status = SiloStatus.Active;
logger.Info("Calling UpdateRow with Entry = {0} eTag = {1} New TableVersion={2}", MembershipEntry, eTag, tableVer);
ok = await membership.UpdateRow(MembershipEntry, eTag, tableVer);
MembershipData = await membership.ReadAll();
Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after update: {0}", MembershipData);
Assert.IsTrue(ok, "UpdateRow OK - Table Data = {0}", MembershipData);
}
private async Task MembershipTable_InsertRow(IMembershipTable membership)
{
MembershipEntry MembershipEntry = CreateMembershipEntryForTest();
MembershipTableData MembershipData = await membership.ReadAll();
Assert.IsNotNull(MembershipData, "Membership Data not null");
Assert.AreEqual(0, MembershipData.Members.Count, "Should be no data initially: {0}", MembershipData);
bool ok = await membership.InsertRow(MembershipEntry, MembershipData.Version);
Assert.IsTrue(ok, "InsertRow OK");
MembershipData = await membership.ReadAll();
Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after insert: {0}", MembershipData);
}
internal static async Task MembershipTable_ReadAll(IMembershipTable membership)
{
var membershipData = await membership.ReadAll();
Assert.IsNotNull(membershipData, "Membership Data not null");
TableVersion tableVersion = membershipData.Version;
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, membershipData);
Assert.AreEqual(0, membershipData.Members.Count, "Number of records returned - no table version row");
string eTag = tableVersion.VersionEtag;
int ver = tableVersion.Version;
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.AreEqual(0, ver, "Initial tabel version should be zero");
}
// Only used with MembershipTableGrain to wiat for primary to start.
internal async Task WaitForTableToInit(IMembershipTable membershipTable)
{
var timespan = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(5);
// This is a quick temporary solution to enable primary node to start fully before secondaries.
// Secondary silos waits untill GrainBasedMembershipTable is created.
for (int i = 0; i < 100; i++)
{
bool needToWait = false;
try
{
MembershipTableData table = await membershipTable.ReadAll().WithTimeout(timespan);
if (table.Members.Any(tuple => tuple.Item1.IsPrimary))
{
logger.Info(ErrorCode.MembershipTableGrainInit1,
"-Connected to membership table provider and also found primary silo registered in the table.");
return;
}
logger.Info(ErrorCode.MembershipTableGrainInit2,
"-Connected to membership table provider but did not find primary silo registered in the table. Going to try again for a {0}th time.", i);
}
catch (Exception exc)
{
var type = exc.GetBaseException().GetType();
if (type == typeof(TimeoutException) || type == typeof(OrleansException))
{
logger.Info(ErrorCode.MembershipTableGrainInit3,
"-Waiting for membership table provider to initialize. Going to sleep for {0} and re-try to reconnect.", timespan);
needToWait = true;
}
else
{
logger.Info(ErrorCode.MembershipTableGrainInit4, "-Membership table provider failed to initialize. Giving up.");
throw;
}
}
if (needToWait)
{
await Task.Delay(timespan);
}
}
}
internal static async Task MembershipTable_ReadAll(IMembershipTable membership)
{
var membershipData = await membership.ReadAll();
Assert.IsNotNull(membershipData, "Membership Data not null");
TableVersion tableVersion = membershipData.Version;
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, membershipData);
Assert.AreEqual(0, membershipData.Members.Count, "Number of records returned - no table version row");
string eTag = tableVersion.VersionEtag;
int ver = tableVersion.Version;
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.AreEqual(0, ver, "Initial tabel version should be zero");
}
// Only used with MembershipTableGrain to wait for primary to start.
internal async Task WaitForTableToInit(IMembershipTable membershipTable)
{
var timespan = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(5);
// This is a quick temporary solution to enable primary node to start fully before secondaries.
// Secondary silos waits untill GrainBasedMembershipTable is created.
for (int i = 0; i < 100; i++)
{
bool needToWait = false;
try
{
MembershipTableData table = await membershipTable.ReadAll().WithTimeout(timespan);
logger.Info(ErrorCode.MembershipTableGrainInit2, "-Connected to membership table provider.");
return;
}
catch (Exception exc)
{
var type = exc.GetBaseException().GetType();
if (type == typeof(TimeoutException) || type == typeof(OrleansException))
{
logger.Info(ErrorCode.MembershipTableGrainInit3,
"-Waiting for membership table provider to initialize. Going to sleep for {0} and re-try to reconnect.", timespan);
needToWait = true;
}
else
{
logger.Info(ErrorCode.MembershipTableGrainInit4, "-Membership table provider failed to initialize. Giving up.");
throw;
}
}
if (needToWait)
{
await Task.Delay(timespan);
}
}
}
private async Task MembershipTable_UpdateRow(IMembershipTable membership)
{
MembershipEntry MembershipEntry = CreateMembershipEntryForTest();
MembershipTableData MembershipData = await membership.ReadAll();
TableVersion tableVer = MembershipData.Version;
Assert.AreEqual(0, MembershipData.Members.Count, "Should be no data initially: {0}", MembershipData);
logger.Info("Calling InsertRow with Entry = {0} TableVersion = {1}", MembershipEntry, tableVer);
bool ok = await membership.InsertRow(MembershipEntry, tableVer);
Assert.IsTrue(ok, "InsertRow OK");
MembershipData = await membership.ReadAll();
Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after insert: {0}", MembershipData);
Tuple<MembershipEntry, string> newEntryData = MembershipData.Get(MembershipEntry.SiloAddress);
string eTag = newEntryData.Item2;
Assert.IsNotNull(eTag, "ETag should not be null");
tableVer = MembershipData.Version;
Assert.IsNotNull(tableVer, "TableVersion should not be null");
tableVer = tableVer.Next();
MembershipEntry = CreateMembershipEntryForTest();
MembershipEntry.Status = SiloStatus.Active;
logger.Info("Calling UpdateRow with Entry = {0} eTag = {1} New TableVersion={2}", MembershipEntry, eTag, tableVer);
ok = await membership.UpdateRow(MembershipEntry, eTag, tableVer);
MembershipData = await membership.ReadAll();
Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after update: {0}", MembershipData);
Assert.IsTrue(ok, "UpdateRow OK - Table Data = {0}", MembershipData);
}
// Test function methods
private async Task MembershipTable_ReadAll(IMembershipTable membership)
{
MembershipTableData MembershipData = await membership.ReadAll();
Assert.IsNotNull(MembershipData, "Membership Data not null");
}
private async Task MembershipTable_ReadAll(IMembershipTable membership)
{
var membershipData = await membership.ReadAll();
Assert.IsNotNull(membershipData, "Membership Data not null");
}
internal static async Task MembershipTable_UpdateRow(IMembershipTable membership)
{
MembershipEntry data = CreateMembershipEntryForTest();
MembershipTableData tableData = await membership.ReadAll();
//TableVersion tableVer = tableData.Version;
Assert.IsNotNull(tableData.Version, "TableVersion should not be null");
Assert.AreEqual(0, tableData.Version.Version, "TableVersion should be zero");
Assert.AreEqual(0, tableData.Members.Count, "Should be no data initially: {0}", tableData);
TableVersion newTableVer = tableData.Version.Next();
logger.Info("Calling InsertRow with Entry = {0} TableVersion = {1}", data, newTableVer);
bool ok = await membership.InsertRow(data, newTableVer);
Assert.IsTrue(ok, "InsertRow OK");
tableData = await membership.ReadAll();
Assert.IsNotNull(tableData.Version, "TableVersion should not be null");
Assert.AreEqual(1, tableData.Version.Version, "TableVersion should be 1");
Assert.AreEqual(1, tableData.Members.Count, "Should be one row after insert: {0}", tableData);
Tuple<MembershipEntry, string> insertedData = tableData.Get(data.SiloAddress);
Assert.IsNotNull(insertedData.Item2, "ETag should not be null");
insertedData.Item1.Status = SiloStatus.Active;
newTableVer = tableData.Version.Next();
logger.Info("Calling UpdateRow with Entry = {0} eTag = {1} New TableVersion={2}", insertedData.Item1, insertedData.Item2, newTableVer);
ok = await membership.UpdateRow(insertedData.Item1, insertedData.Item2, newTableVer);
tableData = await membership.ReadAll();
Assert.IsNotNull(tableData.Version, "TableVersion should not be null");
Assert.AreEqual(2, tableData.Version.Version, "TableVersion should be 2");
Assert.AreEqual(1, tableData.Members.Count, "Should be one row after insert: {0}", tableData);
Assert.IsTrue(ok, "UpdateRow OK - Table Data = {0}", tableData);
}
internal static async Task MembershipTable_ReadRow_Insert_Read(IMembershipTable membership, SiloAddress siloAddress)
{
MembershipTableData data = await membership.ReadAll();
TableVersion tableVersion = data.Version;
logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, data);
Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row");
DateTime now = DateTime.UtcNow;
MembershipEntry entry = new MembershipEntry
{
SiloAddress = siloAddress,
StartTime = now,
Status = SiloStatus.Active,
};
TableVersion newTableVersion = tableVersion.Next();
bool ok = await membership.InsertRow(entry, newTableVersion);
Assert.IsTrue(ok, "InsertRow completed successfully");
data = await membership.ReadRow(siloAddress);
tableVersion = data.Version;
logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", tableVersion, data);
Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only");
Assert.IsNotNull(tableVersion.VersionEtag, "New version ETag should not be null");
Assert.AreNotEqual(newTableVersion.VersionEtag, tableVersion.VersionEtag, "New VersionEtag differetnfrom last");
Assert.AreEqual(newTableVersion.Version, tableVersion.Version, "New table version number");
MembershipEntry MembershipEntry = data.Members[0].Item1;
string eTag = data.Members[0].Item2;
logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry);
Assert.IsNotNull(eTag, "ETag should not be null");
Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null");
}