internal static async Task MembershipTable_ReadAll_Insert_ReadAll(IMembershipTable membership) { MembershipTableData data = await membership.ReadAll(); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row"); TableVersion newTableVersion = data.Version.Next(); MembershipEntry newEntry = CreateMembershipEntryForTest(); bool ok = await membership.InsertRow(newEntry, newTableVersion); Assert.IsTrue(ok, "InsertRow completed successfully"); data = await membership.ReadAll(); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only"); Assert.IsNotNull(data.Version.VersionEtag, "New version ETag should not be null"); Assert.AreNotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag, "New VersionEtag differetnfrom last"); Assert.AreEqual(newTableVersion.Version, data.Version.Version, "New table version number"); MembershipEntry MembershipEntry = data.Members[0].Item1; string eTag = data.Members[0].Item2; logger.Info("Membership.ReadAll returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry); Assert.IsNotNull(eTag, "ETag should not be null"); Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null"); }
// read the table // find all currently active nodes and test pings to all of them // try to ping all // if all pings succeeded // try to change my status to Active and in the same write transaction update Membership version row, conditioned on both etags // if failed (on ping or on write exception or on etag) - retry the whole AttemptToJoinActiveNodes private async Task <bool> TryUpdateMyStatusGlobalOnce(SiloStatus newStatus) { var table = await membershipTableProvider.ReadAll(); if (log.IsEnabled(LogLevel.Debug)) { log.LogDebug( "TryUpdateMyStatusGlobalOnce: Read{Selection} Membership table {Table}", (newStatus.Equals(SiloStatus.Active) ? "All" : " my entry from"), table.ToString()); } LogMissedIAmAlives(table); var(myEntry, myEtag) = this.GetOrCreateLocalSiloEntry(table, newStatus); if (myEntry.Status == SiloStatus.Dead && myEntry.Status != newStatus) { this.log.LogWarning( (int)ErrorCode.MembershipFoundMyselfDead1, "I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {Entry}.", myEntry.ToFullString(full: true)); this.KillMyselfLocally($"I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {(myEntry.ToFullString(full: true))}."); return(true); } var now = GetDateTimeUtcNow(); if (newStatus == SiloStatus.Dead) { myEntry.AddSuspector(myAddress, now); // add the killer (myself) to the suspect list, for easier diagnostics later on. } myEntry.Status = newStatus; myEntry.IAmAliveTime = now; bool ok; TableVersion next = table.Version.Next(); if (myEtag != null) // no previous etag for my entry -> its the first write to this entry, so insert instead of update. { ok = await membershipTableProvider.UpdateRow(myEntry, myEtag, next); } else { ok = await membershipTableProvider.InsertRow(myEntry, next); } if (ok) { this.CurrentStatus = newStatus; var entries = table.Members.ToDictionary(e => e.Item1.SiloAddress, e => e); entries[myEntry.SiloAddress] = Tuple.Create(myEntry, myEtag); var updatedTable = new MembershipTableData(entries.Values.ToList(), next); this.ProcessTableUpdate(updatedTable, nameof(TryUpdateMyStatusGlobalOnce)); } return(ok); }
internal static async Task MembershipTable_InsertRow(IMembershipTable membership) { var membershipEntry = CreateMembershipEntryForTest(); var data = await membership.ReadAll(); Assert.IsNotNull(data, "Membership Data not null"); Assert.AreEqual(0, data.Members.Count, "Should be no data initially: {0}", data); bool ok = await membership.InsertRow(membershipEntry, data.Version.Next()); Assert.IsTrue(ok, "InsertRow OK"); data = await membership.ReadAll(); Assert.AreEqual(1, data.Members.Count, "Should be one row after insert: {0}", data); }
internal static async Task MembershipTable_ReadRow_Insert_Read(IMembershipTable membership) { MembershipTableData data = await membership.ReadAll(); //TableVersion tableVersion = data.Version; logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row"); TableVersion newTableVersion = data.Version.Next(); MembershipEntry newEntry = CreateMembershipEntryForTest(); bool ok = await membership.InsertRow(newEntry, newTableVersion); Assert.IsTrue(ok, "InsertRow completed successfully"); data = await membership.ReadRow(newEntry.SiloAddress); logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", data.Version, data); Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only"); Assert.IsNotNull(data.Version.VersionEtag, "New version ETag should not be null"); Assert.AreNotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag, "New VersionEtag differetnfrom last"); Assert.AreEqual(newTableVersion.Version, data.Version.Version, "New table version number"); MembershipEntry MembershipEntry = data.Members[0].Item1; string eTag = data.Members[0].Item2; logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry); Assert.IsNotNull(eTag, "ETag should not be null"); Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null"); }
private async Task MembershipTable_InsertRow(IMembershipTable membership) { MembershipEntry MembershipEntry = CreateMembershipEntryForTest(); MembershipTableData MembershipData = await membership.ReadAll(); Assert.IsNotNull(MembershipData, "Membership Data not null"); Assert.AreEqual(0, MembershipData.Members.Count, "Should be no data initially: {0}", MembershipData); bool ok = await membership.InsertRow(MembershipEntry, MembershipData.Version); Assert.IsTrue(ok, "InsertRow OK"); MembershipData = await membership.ReadAll(); Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after insert: {0}", MembershipData); }
internal static async Task MembershipTable_UpdateRow(IMembershipTable membership) { MembershipEntry data = CreateMembershipEntryForTest(); MembershipTableData tableData = await membership.ReadAll(); //TableVersion tableVer = tableData.Version; Assert.IsNotNull(tableData.Version, "TableVersion should not be null"); Assert.AreEqual(0, tableData.Version.Version, "TableVersion should be zero"); Assert.AreEqual(0, tableData.Members.Count, "Should be no data initially: {0}", tableData); TableVersion newTableVer = tableData.Version.Next(); logger.Info("Calling InsertRow with Entry = {0} TableVersion = {1}", data, newTableVer); bool ok = await membership.InsertRow(data, newTableVer); Assert.IsTrue(ok, "InsertRow OK"); tableData = await membership.ReadAll(); Assert.IsNotNull(tableData.Version, "TableVersion should not be null"); Assert.AreEqual(1, tableData.Version.Version, "TableVersion should be 1"); Assert.AreEqual(1, tableData.Members.Count, "Should be one row after insert: {0}", tableData); Tuple <MembershipEntry, string> insertedData = tableData.Get(data.SiloAddress); Assert.IsNotNull(insertedData.Item2, "ETag should not be null"); insertedData.Item1.Status = SiloStatus.Active; newTableVer = tableData.Version.Next(); logger.Info("Calling UpdateRow with Entry = {0} eTag = {1} New TableVersion={2}", insertedData.Item1, insertedData.Item2, newTableVer); ok = await membership.UpdateRow(insertedData.Item1, insertedData.Item2, newTableVer); tableData = await membership.ReadAll(); Assert.IsNotNull(tableData.Version, "TableVersion should not be null"); Assert.AreEqual(2, tableData.Version.Version, "TableVersion should be 2"); Assert.AreEqual(1, tableData.Members.Count, "Should be one row after insert: {0}", tableData); Assert.IsTrue(ok, "UpdateRow OK - Table Data = {0}", tableData); }
internal static async Task MembershipTable_ReadAll_Insert_ReadAll(IMembershipTable membership, SiloAddress siloAddress) { MembershipTableData data = await membership.ReadAll(); TableVersion tableVersion = data.Version; logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, data); Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row"); DateTime now = DateTime.UtcNow; MembershipEntry entry = new MembershipEntry { SiloAddress = siloAddress, StartTime = now, Status = SiloStatus.Active, }; TableVersion newTableVersion = tableVersion.Next(); bool ok = await membership.InsertRow(entry, newTableVersion); Assert.IsTrue(ok, "InsertRow completed successfully"); data = await membership.ReadAll(); tableVersion = data.Version; logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, data); Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only"); Assert.IsNotNull(tableVersion.VersionEtag, "New version ETag should not be null"); Assert.AreNotEqual(newTableVersion.VersionEtag, tableVersion.VersionEtag, "New VersionEtag differetnfrom last"); Assert.AreEqual(newTableVersion.Version, tableVersion.Version, "New table version number"); MembershipEntry MembershipEntry = data.Members[0].Item1; string eTag = data.Members[0].Item2; logger.Info("Membership.ReadAll returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry); Assert.IsNotNull(eTag, "ETag should not be null"); Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null"); }
internal static async Task MembershipTable_ReadAll_EmptyTable(IMembershipTable membership) { var data = await membership.ReadAll(); Assert.IsNotNull(data, "Membership Data not null"); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row"); Assert.IsNotNull(data.Version.VersionEtag, "ETag should not be null"); Assert.AreEqual(0, data.Version.Version, "Initial tabel version should be zero"); }
private async Task MembershipTable_UpdateRow(IMembershipTable membership) { MembershipEntry MembershipEntry = CreateMembershipEntryForTest(); MembershipTableData MembershipData = await membership.ReadAll(); TableVersion tableVer = MembershipData.Version; Assert.AreEqual(0, MembershipData.Members.Count, "Should be no data initially: {0}", MembershipData); logger.Info("Calling InsertRow with Entry = {0} TableVersion = {1}", MembershipEntry, tableVer); bool ok = await membership.InsertRow(MembershipEntry, tableVer); Assert.IsTrue(ok, "InsertRow OK"); MembershipData = await membership.ReadAll(); Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after insert: {0}", MembershipData); Tuple <MembershipEntry, string> newEntryData = MembershipData.Get(MembershipEntry.SiloAddress); string eTag = newEntryData.Item2; Assert.IsNotNull(eTag, "ETag should not be null"); tableVer = MembershipData.Version; Assert.IsNotNull(tableVer, "TableVersion should not be null"); tableVer = tableVer.Next(); MembershipEntry = CreateMembershipEntryForTest(); MembershipEntry.Status = SiloStatus.Active; logger.Info("Calling UpdateRow with Entry = {0} eTag = {1} New TableVersion={2}", MembershipEntry, eTag, tableVer); ok = await membership.UpdateRow(MembershipEntry, eTag, tableVer); MembershipData = await membership.ReadAll(); Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after update: {0}", MembershipData); Assert.IsTrue(ok, "UpdateRow OK - Table Data = {0}", MembershipData); }
internal static async Task MembershipTable_ReadAll(IMembershipTable membership) { var membershipData = await membership.ReadAll(); Assert.IsNotNull(membershipData, "Membership Data not null"); TableVersion tableVersion = membershipData.Version; logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, membershipData); Assert.AreEqual(0, membershipData.Members.Count, "Number of records returned - no table version row"); string eTag = tableVersion.VersionEtag; int ver = tableVersion.Version; Assert.IsNotNull(eTag, "ETag should not be null"); Assert.AreEqual(0, ver, "Initial tabel version should be zero"); }
// Only used with MembershipTableGrain to wiat for primary to start. internal async Task WaitForTableToInit(IMembershipTable membershipTable) { var timespan = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(5); // This is a quick temporary solution to enable primary node to start fully before secondaries. // Secondary silos waits untill GrainBasedMembershipTable is created. for (int i = 0; i < 100; i++) { bool needToWait = false; try { MembershipTableData table = await membershipTable.ReadAll().WithTimeout(timespan); if (table.Members.Any(tuple => tuple.Item1.IsPrimary)) { logger.Info(ErrorCode.MembershipTableGrainInit1, "-Connected to membership table provider and also found primary silo registered in the table."); return; } logger.Info(ErrorCode.MembershipTableGrainInit2, "-Connected to membership table provider but did not find primary silo registered in the table. Going to try again for a {0}th time.", i); } catch (Exception exc) { var type = exc.GetBaseException().GetType(); if (type == typeof(TimeoutException) || type == typeof(OrleansException)) { logger.Info(ErrorCode.MembershipTableGrainInit3, "-Waiting for membership table provider to initialize. Going to sleep for {0} and re-try to reconnect.", timespan); needToWait = true; } else { logger.Info(ErrorCode.MembershipTableGrainInit4, "-Membership table provider failed to initialize. Giving up."); throw; } } if (needToWait) { await Task.Delay(timespan); } } }
// Only used with MembershipTableGrain to wait for primary to start. internal async Task WaitForTableToInit(IMembershipTable membershipTable) { var timespan = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(5); // This is a quick temporary solution to enable primary node to start fully before secondaries. // Secondary silos waits untill GrainBasedMembershipTable is created. for (int i = 0; i < 100; i++) { bool needToWait = false; try { MembershipTableData table = await membershipTable.ReadAll().WithTimeout(timespan); logger.Info(ErrorCode.MembershipTableGrainInit2, "-Connected to membership table provider."); return; } catch (Exception exc) { var type = exc.GetBaseException().GetType(); if (type == typeof(TimeoutException) || type == typeof(OrleansException)) { logger.Info(ErrorCode.MembershipTableGrainInit3, "-Waiting for membership table provider to initialize. Going to sleep for {0} and re-try to reconnect.", timespan); needToWait = true; } else { logger.Info(ErrorCode.MembershipTableGrainInit4, "-Membership table provider failed to initialize. Giving up."); throw; } } if (needToWait) { await Task.Delay(timespan); } } }
private async Task MembershipTable_UpdateRow(IMembershipTable membership) { MembershipEntry MembershipEntry = CreateMembershipEntryForTest(); MembershipTableData MembershipData = await membership.ReadAll(); TableVersion tableVer = MembershipData.Version; Assert.AreEqual(0, MembershipData.Members.Count, "Should be no data initially: {0}", MembershipData); logger.Info("Calling InsertRow with Entry = {0} TableVersion = {1}", MembershipEntry, tableVer); bool ok = await membership.InsertRow(MembershipEntry, tableVer); Assert.IsTrue(ok, "InsertRow OK"); MembershipData = await membership.ReadAll(); Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after insert: {0}", MembershipData); Tuple<MembershipEntry, string> newEntryData = MembershipData.Get(MembershipEntry.SiloAddress); string eTag = newEntryData.Item2; Assert.IsNotNull(eTag, "ETag should not be null"); tableVer = MembershipData.Version; Assert.IsNotNull(tableVer, "TableVersion should not be null"); tableVer = tableVer.Next(); MembershipEntry = CreateMembershipEntryForTest(); MembershipEntry.Status = SiloStatus.Active; logger.Info("Calling UpdateRow with Entry = {0} eTag = {1} New TableVersion={2}", MembershipEntry, eTag, tableVer); ok = await membership.UpdateRow(MembershipEntry, eTag, tableVer); MembershipData = await membership.ReadAll(); Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after update: {0}", MembershipData); Assert.IsTrue(ok, "UpdateRow OK - Table Data = {0}", MembershipData); }
// Test function methods private async Task MembershipTable_ReadAll(IMembershipTable membership) { MembershipTableData MembershipData = await membership.ReadAll(); Assert.IsNotNull(MembershipData, "Membership Data not null"); }
private async Task MembershipTable_ReadAll(IMembershipTable membership) { var membershipData = await membership.ReadAll(); Assert.IsNotNull(membershipData, "Membership Data not null"); }
internal static async Task MembershipTable_UpdateRow(IMembershipTable membership) { MembershipEntry data = CreateMembershipEntryForTest(); MembershipTableData tableData = await membership.ReadAll(); //TableVersion tableVer = tableData.Version; Assert.IsNotNull(tableData.Version, "TableVersion should not be null"); Assert.AreEqual(0, tableData.Version.Version, "TableVersion should be zero"); Assert.AreEqual(0, tableData.Members.Count, "Should be no data initially: {0}", tableData); TableVersion newTableVer = tableData.Version.Next(); logger.Info("Calling InsertRow with Entry = {0} TableVersion = {1}", data, newTableVer); bool ok = await membership.InsertRow(data, newTableVer); Assert.IsTrue(ok, "InsertRow OK"); tableData = await membership.ReadAll(); Assert.IsNotNull(tableData.Version, "TableVersion should not be null"); Assert.AreEqual(1, tableData.Version.Version, "TableVersion should be 1"); Assert.AreEqual(1, tableData.Members.Count, "Should be one row after insert: {0}", tableData); Tuple<MembershipEntry, string> insertedData = tableData.Get(data.SiloAddress); Assert.IsNotNull(insertedData.Item2, "ETag should not be null"); insertedData.Item1.Status = SiloStatus.Active; newTableVer = tableData.Version.Next(); logger.Info("Calling UpdateRow with Entry = {0} eTag = {1} New TableVersion={2}", insertedData.Item1, insertedData.Item2, newTableVer); ok = await membership.UpdateRow(insertedData.Item1, insertedData.Item2, newTableVer); tableData = await membership.ReadAll(); Assert.IsNotNull(tableData.Version, "TableVersion should not be null"); Assert.AreEqual(2, tableData.Version.Version, "TableVersion should be 2"); Assert.AreEqual(1, tableData.Members.Count, "Should be one row after insert: {0}", tableData); Assert.IsTrue(ok, "UpdateRow OK - Table Data = {0}", tableData); }
internal static async Task MembershipTable_ReadRow_Insert_Read(IMembershipTable membership, SiloAddress siloAddress) { MembershipTableData data = await membership.ReadAll(); TableVersion tableVersion = data.Version; logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, data); Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row"); DateTime now = DateTime.UtcNow; MembershipEntry entry = new MembershipEntry { SiloAddress = siloAddress, StartTime = now, Status = SiloStatus.Active, }; TableVersion newTableVersion = tableVersion.Next(); bool ok = await membership.InsertRow(entry, newTableVersion); Assert.IsTrue(ok, "InsertRow completed successfully"); data = await membership.ReadRow(siloAddress); tableVersion = data.Version; logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", tableVersion, data); Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only"); Assert.IsNotNull(tableVersion.VersionEtag, "New version ETag should not be null"); Assert.AreNotEqual(newTableVersion.VersionEtag, tableVersion.VersionEtag, "New VersionEtag differetnfrom last"); Assert.AreEqual(newTableVersion.Version, tableVersion.Version, "New table version number"); MembershipEntry MembershipEntry = data.Members[0].Item1; string eTag = data.Members[0].Item2; logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry); Assert.IsNotNull(eTag, "ETag should not be null"); Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null"); }