protected async Task MembershipTable_ReadAll_Insert_ReadAll() { MembershipTableData data = await membershipTable.ReadAll(); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.Equal(0, data.Members.Count); TableVersion newTableVersion = data.Version.Next(); MembershipEntry newEntry = CreateMembershipEntryForTest(); bool ok = await membershipTable.InsertRow(newEntry, newTableVersion); Assert.True(ok, "InsertRow failed"); data = await membershipTable.ReadAll(); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.Equal(1, data.Members.Count); Assert.NotNull(data.Version.VersionEtag); Assert.NotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag); Assert.Equal(newTableVersion.Version, data.Version.Version); var membershipEntry = data.Members[0].Item1; string eTag = data.Members[0].Item2; logger.Info("Membership.ReadAll returned MembershipEntry ETag={0} Entry={1}", eTag, membershipEntry); Assert.NotNull(eTag); Assert.NotNull(membershipEntry); }
internal static async Task MembershipTable_ReadAll_Insert_ReadAll(IMembershipTable membership) { MembershipTableData data = await membership.ReadAll(); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row"); TableVersion newTableVersion = data.Version.Next(); MembershipEntry newEntry = CreateMembershipEntryForTest(); bool ok = await membership.InsertRow(newEntry, newTableVersion); Assert.IsTrue(ok, "InsertRow completed successfully"); data = await membership.ReadAll(); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only"); Assert.IsNotNull(data.Version.VersionEtag, "New version ETag should not be null"); Assert.AreNotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag, "New VersionEtag differetnfrom last"); Assert.AreEqual(newTableVersion.Version, data.Version.Version, "New table version number"); MembershipEntry MembershipEntry = data.Members[0].Item1; string eTag = data.Members[0].Item2; logger.Info("Membership.ReadAll returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry); Assert.IsNotNull(eTag, "ETag should not be null"); Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null"); }
public async Task <MembershipTableData> ReadRow(SiloAddress key) { try { MembershipTableData data = null; var val = await database.StringGetAsync(clusterId); if (!val.IsNull) { RedisMembershipCollection collection = serializer.Deserialize <RedisMembershipCollection>(val); data = collection.ToMembershipTableData(); } else { data = new MembershipTableData(_tableVersion); } return(data); } catch (Exception ex) { logger?.LogError(ex, "Redis membership table key '{0}' failed read row.", clusterId); throw ex; } }
protected async Task MembershipTable_UpdateIAmAlive(bool extendedProtocol = true) { MembershipTableData tableData = await this.membershipTable.ReadAll(); TableVersion newTableVersion = tableData.Version.Next(); MembershipEntry newEntry = CreateMembershipEntryForTest(); bool ok = await this.membershipTable.InsertRow(newEntry, newTableVersion); Assert.True(ok); var amAliveTime = DateTime.UtcNow; // This mimics the arguments MembershipOracle.OnIAmAliveUpdateInTableTimer passes in var entry = new MembershipEntry { SiloAddress = newEntry.SiloAddress, IAmAliveTime = amAliveTime }; await this.membershipTable.UpdateIAmAlive(entry); tableData = await this.membershipTable.ReadAll(); Tuple <MembershipEntry, string> member = tableData.Members.First(); // compare that the value is close to what we passed in, but not exactly, as the underlying store can set its own precision settings // (ie: in SQL Server this is defined as datetime2(3), so we don't expect precision to account for less than 0.001s values) Assert.True((amAliveTime - member.Item1.IAmAliveTime).Duration() < TimeSpan.FromMilliseconds(50), (amAliveTime - member.Item1.IAmAliveTime).Duration().ToString()); }
private MembershipTableData Convert(List<SiloInstanceRecord> entries) { try { var memEntries = new List<Tuple<MembershipEntry, string>>(); foreach (var tableEntry in entries) { try { MembershipEntry membershipEntry = Parse(tableEntry); memEntries.Add(new Tuple<MembershipEntry, string>(membershipEntry, tableEntry.ETag.ToString())); } catch (Exception exc) { logger.Error(ErrorCode.MembershipBase, $"Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {tableEntry}. Ignoring this entry.", exc); } } var data = new MembershipTableData(memEntries, _tableVersion); return data; } catch (Exception exc) { logger.Error(ErrorCode.MembershipBase, $"Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {Utils.EnumerableToString(entries, e => e.ToString())}.", exc); throw; } }
public async Task <MembershipTableData> ReadRow(SiloAddress siloAddress) { try { var keys = new Dictionary <string, AttributeValue> { { $"{SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME}", new AttributeValue(this.clusterId) }, { $"{SiloInstanceRecord.SILO_IDENTITY_PROPERTY_NAME}", new AttributeValue(SiloInstanceRecord.ConstructSiloIdentity(siloAddress)) } }; var entry = await storage.ReadSingleEntryAsync(this.options.TableName, keys, fields => new SiloInstanceRecord(fields)); MembershipTableData data = entry != null?Convert(new List <SiloInstanceRecord> { entry }) : new MembershipTableData(this.tableVersion); if (this.logger.IsEnabled(LogLevel.Trace)) { this.logger.Trace("Read my entry {0} Table=" + Environment.NewLine + "{1}", siloAddress.ToLongString(), data.ToString()); } return(data); } catch (Exception exc) { this.logger.Warn(ErrorCode.MembershipBase, $"Intermediate error reading silo entry for key {siloAddress.ToLongString()} from the table {this.options.TableName}.", exc); throw; } }
private void LogMissedIAmAlives(MembershipTableData table) { foreach (var pair in table.Members) { var entry = pair.Item1; if (entry.SiloAddress.Equals(myAddress)) { continue; } if (entry.Status != SiloStatus.Active) { continue; } var now = GetDateTimeUtcNow(); var missedSince = entry.HasMissedIAmAlivesSince(this.clusterMembershipOptions, now); if (missedSince != null) { log.LogWarning( (int)ErrorCode.MembershipMissedIAmAliveTableUpdate, "Noticed that silo {SiloAddress} has not updated it's IAmAliveTime table column recently." + " Last update was at {LastUpdateTime}, now is {CurrentTime}, no update for {SinceUpdate}, which is more than {AllowedIAmAliveMissPeriod}.", entry.SiloAddress, missedSince, now, now - missedSince, clusterMembershipOptions.AllowedIAmAliveMissPeriod); } } }
private void DetectNodeMigration(MembershipTableData table, string myHostname) { string mySiloName = this.localSiloDetails.Name; MembershipEntry mostRecentPreviousEntry = null; // look for silo instances that are same as me, find most recent with Generation before me. foreach (var entry in table.Members.Select(tuple => tuple.Item1).Where(data => mySiloName.Equals(data.SiloName))) { bool iAmLater = myAddress.Generation.CompareTo(entry.SiloAddress.Generation) > 0; // more recent if (iAmLater && (mostRecentPreviousEntry == null || entry.SiloAddress.Generation.CompareTo(mostRecentPreviousEntry.SiloAddress.Generation) > 0)) { mostRecentPreviousEntry = entry; } } if (mostRecentPreviousEntry != null) { bool physicalHostChanged = !myHostname.Equals(mostRecentPreviousEntry.HostName) || !myAddress.Endpoint.Equals(mostRecentPreviousEntry.SiloAddress.Endpoint); if (physicalHostChanged) { string error = string.Format("Silo {0} migrated from host {1} silo address {2} to host {3} silo address {4}.", mySiloName, myHostname, myAddress, mostRecentPreviousEntry.HostName, mostRecentPreviousEntry.SiloAddress); log.Warn(ErrorCode.MembershipNodeMigrated, error); } else { string error = string.Format("Silo {0} restarted on same host {1} New silo address = {2} Previous silo address = {3}", mySiloName, myHostname, myAddress, mostRecentPreviousEntry.SiloAddress); log.Warn(ErrorCode.MembershipNodeRestarted, error); } } }
private void ProcessTableUpdate(MembershipTableData table, string caller) { if (table is null) { throw new ArgumentNullException(nameof(table)); } if (this.log.IsEnabled(LogLevel.Debug)) { this.log.LogDebug(nameof(ProcessTableUpdate) + " (called from {Caller}) membership table {Table}", caller, table.ToString()); } // Update the current membership snapshot. var(localSiloEntry, _) = this.GetOrCreateLocalSiloEntry(table, this.CurrentStatus); var updated = MembershipTableSnapshot.Create(localSiloEntry, table); if (this.updates.TryPublish(updated)) { this.LogMissedIAmAlives(table); this.log.LogInformation( (int)ErrorCode.MembershipReadAll_2, nameof(ProcessTableUpdate) + " (called from {Caller}) membership table: {Table}", caller, table.WithoutDuplicateDeads().ToString()); } }
private void LogMissedIAmAlives(MembershipTableData table) { foreach (var pair in table.Members) { var entry = pair.Item1; if (entry.SiloAddress.Equals(myAddress)) { continue; } if (entry.Status != SiloStatus.Active) { continue; } var now = DateTime.UtcNow; var missedSince = entry.HasMissedIAmAlivesSince(this.clusterMembershipOptions, now); if (missedSince != null) { log.Warn( ErrorCode.MembershipMissedIAmAliveTableUpdate, $"Noticed that silo {entry.SiloAddress} has not updated it's IAmAliveTime table column recently." + $" Last update was at {missedSince}, now is {now}, no update for {now - missedSince}, which is more than {this.clusterMembershipOptions.AllowedIAmAliveMissPeriod}."); } } }
public async Task <MembershipTableData> ReadAll() { var entries = new List <Tuple <MembershipEntry, string> >(); TableVersion version = await ReadTableVersion(); try { var recordSet = _client.Query(null, new Statement() { Filter = Filter.Equal("clusterid", _clusterOptions.ClusterId), Namespace = _options.Namespace, SetName = _options.SetName }); while (recordSet.Next()) { entries.Add( new Tuple <MembershipEntry, string>( ParseMembershipEntryRecord(recordSet.Record), recordSet.Record.generation.ToString())); } } catch (Exception exp) { } var data = new MembershipTableData(entries, version); return(data); }
public static MembershipTableSnapshot Create(MembershipEntry localSiloEntry, MembershipTableData table) { if (table is null) { throw new ArgumentNullException(nameof(table)); } var entries = ImmutableDictionary.CreateBuilder <SiloAddress, MembershipEntry>(); if (table.Members != null) { foreach (var item in table.Members) { var entry = item.Item1; entries.Add(entry.SiloAddress, entry); } } if (entries.TryGetValue(localSiloEntry.SiloAddress, out var existing)) { entries[localSiloEntry.SiloAddress] = existing.WithStatus(localSiloEntry.Status); } else { entries[localSiloEntry.SiloAddress] = localSiloEntry; } var version = new MembershipVersion(table.Version.Version); return(new MembershipTableSnapshot(version, entries.ToImmutable())); }
protected async Task MembershipTable_UpdateIAmAlive(bool extendedProtocol = true) { MembershipTableData tableData = await membershipTable.ReadAll(); TableVersion newTableVersion = tableData.Version.Next(); MembershipEntry newEntry = CreateMembershipEntryForTest(); bool ok = await membershipTable.InsertRow(newEntry, newTableVersion); Assert.True(ok); var amAliveTime = DateTime.UtcNow; // This mimics the arguments MembershipOracle.OnIAmAliveUpdateInTableTimer passes in var entry = new MembershipEntry { SiloAddress = newEntry.SiloAddress, IAmAliveTime = amAliveTime }; await membershipTable.UpdateIAmAlive(entry); tableData = await membershipTable.ReadAll(); Tuple <MembershipEntry, string> member = tableData.Members.First(); Assert.True((amAliveTime - member.Item1.IAmAliveTime).Duration() < TimeSpan.FromMilliseconds(1)); }
public async Task <MembershipTableData> ReadRow(SiloAddress key) { var name = ConstructSiloEntityId(key); try { var versionEntity = await this.GetClusterVersion(); SiloEntity entity = default; try { entity = ((JObject)await this._kubeClient.GetNamespacedCustomObjectAsync( Constants.ORLEANS_GROUP, Constants.PROVIDER_MODEL_VERSION, this._namespace, SiloEntity.PLURAL, name ))?.ToObject <SiloEntity>(); } catch (HttpOperationException ex) { if (ex.Response.StatusCode != HttpStatusCode.NotFound) { throw; } } var version = default(TableVersion); if (versionEntity != null) { version = new TableVersion(versionEntity.ClusterVersion, versionEntity.Metadata.ResourceVersion); } else { this._logger?.LogError("Initial ClusterVersionEntity entity doesn't exist."); } var memEntries = new List <Tuple <MembershipEntry, string> >(); if (entity != null) { var membershipEntry = ParseEntity(entity); memEntries.Add(new Tuple <MembershipEntry, string>(membershipEntry, entity.Metadata.ResourceVersion)); } var data = new MembershipTableData(memEntries, version); return(data); } catch (Exception exc) { this._logger?.LogError(exc, "Failure reading silo entry {name} for cluster id {clusterId}.", name, this._clusterOptions.ClusterId); throw; } }
public async Task GatewaySelection_SqlServer() { string testName = Guid.NewGuid().ToString();// TestContext.TestName; Guid serviceId = Guid.NewGuid(); GlobalConfiguration cfg = new GlobalConfiguration { ServiceId = serviceId, DeploymentId = testName, DataConnectionString = TestHelper.TestUtils.GetSqlConnectionString() }; var membership = new SqlMembershipTable(); var logger = LogManager.GetLogger(membership.GetType().Name); await membership.InitializeMembershipTable(cfg, true, logger); IMembershipTable membershipTable = membership; // Pre-populate gateway table with data int count = 1; foreach (Uri gateway in gatewayAddressUris) { output.WriteLine("Adding gataway data for {0}", gateway); SiloAddress siloAddress = gateway.ToSiloAddress(); Assert.IsNotNull(siloAddress, "Unable to get SiloAddress from Uri {0}", gateway); MembershipEntry MembershipEntry = new MembershipEntry { SiloAddress = siloAddress, HostName = gateway.Host, Status = SiloStatus.Active, ProxyPort = gateway.Port, StartTime = DateTime.UtcNow }; var tableVersion = new TableVersion(count, Guid.NewGuid().ToString()); output.WriteLine("Inserting gataway data for {0} with TableVersion={1}", MembershipEntry, tableVersion); bool ok = await membershipTable.InsertRow(MembershipEntry, tableVersion); count++; Assert.IsTrue(ok, "Membership record should have been written OK but were not: {0}", MembershipEntry); output.WriteLine("Successfully inserted Membership row {0}", MembershipEntry); } MembershipTableData data = await membershipTable.ReadAll(); Assert.IsNotNull(data, "MembershipTableData returned"); Assert.AreEqual(gatewayAddressUris.Count, data.Members.Count, "Number of gateway records read"); IGatewayListProvider listProvider = membership; Test_GatewaySelection(listProvider); }
protected async Task MembershipTable_ReadRow_Insert_Read() { MembershipTableData data = await membershipTable.ReadAll(); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.Equal(0, data.Members.Count); TableVersion newTableVersion = data.Version.Next(); MembershipEntry newEntry = CreateMembershipEntryForTest(); bool ok = await membershipTable.InsertRow(newEntry, newTableVersion); Assert.True(ok, "InsertRow failed"); ok = await membershipTable.InsertRow(newEntry, newTableVersion); Assert.False(ok, "InsertRow should have failed - same entry, old table version"); ok = await membershipTable.InsertRow(CreateMembershipEntryForTest(), newTableVersion); Assert.False(ok, "InsertRow should have failed - new entry, old table version"); data = await membershipTable.ReadAll(); Assert.Equal(1, data.Version.Version); var nextTableVersion = data.Version.Next(); ok = await membershipTable.InsertRow(newEntry, nextTableVersion); Assert.False(ok, "InsertRow should have failed - duplicate entry"); data = await membershipTable.ReadAll(); Assert.Equal(1, data.Members.Count); data = await membershipTable.ReadRow(newEntry.SiloAddress); Assert.Equal(newTableVersion.Version, data.Version.Version); logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", data.Version, data); Assert.Equal(1, data.Members.Count); Assert.NotNull(data.Version.VersionEtag); Assert.NotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag); Assert.Equal(newTableVersion.Version, data.Version.Version); var membershipEntry = data.Members[0].Item1; string eTag = data.Members[0].Item2; logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} Entry={1}", eTag, membershipEntry); Assert.NotNull(eTag); Assert.NotNull(membershipEntry); }
// read the table // find all currently active nodes and test pings to all of them // try to ping all // if all pings succeeded // try to change my status to Active and in the same write transaction update Membership version row, conditioned on both etags // if failed (on ping or on write exception or on etag) - retry the whole AttemptToJoinActiveNodes private async Task <bool> TryUpdateMyStatusGlobalOnce(SiloStatus newStatus) { var table = await membershipTableProvider.ReadAll(); if (log.IsEnabled(LogLevel.Debug)) { log.LogDebug( "TryUpdateMyStatusGlobalOnce: Read{Selection} Membership table {Table}", (newStatus.Equals(SiloStatus.Active) ? "All" : " my entry from"), table.ToString()); } LogMissedIAmAlives(table); var(myEntry, myEtag) = this.GetOrCreateLocalSiloEntry(table, newStatus); if (myEntry.Status == SiloStatus.Dead && myEntry.Status != newStatus) { this.log.LogWarning( (int)ErrorCode.MembershipFoundMyselfDead1, "I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {Entry}.", myEntry.ToFullString(full: true)); this.KillMyselfLocally($"I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {(myEntry.ToFullString(full: true))}."); return(true); } var now = GetDateTimeUtcNow(); if (newStatus == SiloStatus.Dead) { myEntry.AddSuspector(myAddress, now); // add the killer (myself) to the suspect list, for easier diagnostics later on. } myEntry.Status = newStatus; myEntry.IAmAliveTime = now; bool ok; TableVersion next = table.Version.Next(); if (myEtag != null) // no previous etag for my entry -> its the first write to this entry, so insert instead of update. { ok = await membershipTableProvider.UpdateRow(myEntry, myEtag, next); } else { ok = await membershipTableProvider.InsertRow(myEntry, next); } if (ok) { this.CurrentStatus = newStatus; var entries = table.Members.ToDictionary(e => e.Item1.SiloAddress, e => e); entries[myEntry.SiloAddress] = Tuple.Create(myEntry, myEtag); var updatedTable = new MembershipTableData(entries.Values.ToList(), next); this.ProcessTableUpdate(updatedTable, nameof(TryUpdateMyStatusGlobalOnce)); } return(ok); }
public async Task <MembershipTableData> ReadRow(SiloAddress key) { var id = ConstructSiloEntityId(key); try { var query = this._container .GetItemQueryIterator <dynamic>( new QueryDefinition(READ_ENTRY_QUERY).WithParameter("@siloId", id), requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey(this._clusterOptions.ClusterId) } ); var docs = await query.ReadNextAsync(); var versionDoc = docs.Where(i => i.EntityType == nameof(ClusterVersionEntity)).SingleOrDefault(); ClusterVersionEntity clusterVersionEntity = versionDoc != null?ClusterVersionEntity.FromDocument(versionDoc) : default; var siloEntities = docs.Where(i => i.EntityType == nameof(SiloEntity)).Select <dynamic, SiloEntity>(d => SiloEntity.FromDocument(d)); TableVersion version = null; if (clusterVersionEntity != null) { version = new TableVersion(clusterVersionEntity.ClusterVersion, clusterVersionEntity.ETag); } else { this._logger.LogError("Initial ClusterVersionEntity entity doesn't exist."); } var memEntries = new List <Tuple <MembershipEntry, string> >(); foreach (var entity in siloEntities) { try { MembershipEntry membershipEntry = ParseEntity(entity); memEntries.Add(new Tuple <MembershipEntry, string>(membershipEntry, entity.ETag)); } catch (Exception exc) { this._logger.LogError(exc, "Failure reading membership row."); throw; } } var data = new MembershipTableData(memEntries, version); return(data); } catch (Exception exc) { this._logger.LogWarning($"Failure reading silo entry {id} for cluster id {this._clusterOptions.ClusterId}: {exc}"); throw; } }
private (MembershipEntry Entry, string ETag) GetOrCreateLocalSiloEntry(MembershipTableData table, SiloStatus currentStatus) { if (table.Contains(this.myAddress)) { var myTuple = table.Get(this.myAddress); return(myTuple.Item1.Copy(), myTuple.Item2); } return(this.CreateLocalSiloEntry(currentStatus), null); }
protected async Task MembershipTable_CleanupDefunctSiloEntries(bool extendedProtocol = true) { MembershipTableData data = await membershipTable.ReadAll(); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.Equal(0, data.Members.Count); TableVersion newTableVersion = data.Version.Next(); var oldEntryDead = CreateMembershipEntryForTest(); oldEntryDead.IAmAliveTime = oldEntryDead.IAmAliveTime.AddDays(-10); oldEntryDead.StartTime = oldEntryDead.StartTime.AddDays(-10); oldEntryDead.Status = SiloStatus.Dead; bool ok = await membershipTable.InsertRow(oldEntryDead, newTableVersion); var table = await membershipTable.ReadAll(); Assert.True(ok, "InsertRow Dead failed"); newTableVersion = table.Version.Next(); var oldEntryJoining = CreateMembershipEntryForTest(); oldEntryJoining.IAmAliveTime = oldEntryJoining.IAmAliveTime.AddDays(-10); oldEntryJoining.StartTime = oldEntryJoining.StartTime.AddDays(-10); oldEntryJoining.Status = SiloStatus.Joining; ok = await membershipTable.InsertRow(oldEntryJoining, newTableVersion); table = await membershipTable.ReadAll(); Assert.True(ok, "InsertRow Joining failed"); newTableVersion = table.Version.Next(); var newEntry = CreateMembershipEntryForTest(); ok = await membershipTable.InsertRow(newEntry, newTableVersion); Assert.True(ok, "InsertRow failed"); data = await membershipTable.ReadAll(); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.Equal(3, data.Members.Count); await membershipTable.CleanupDefunctSiloEntries(oldEntryDead.IAmAliveTime.AddDays(3)); data = await membershipTable.ReadAll(); logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.Equal(1, data.Members.Count); }
protected async Task MembershipTable_ReadRow_Insert_Read() { MembershipTableData data = await membershipTable.ReadAll(); //TableVersion tableVersion = data.Version; logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", data.Version, data); Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row"); TableVersion newTableVersion = data.Version.Next(); MembershipEntry newEntry = CreateMembershipEntryForTest(); bool ok = await membershipTable.InsertRow(newEntry, newTableVersion); Assert.IsTrue(ok, "InsertRow failed"); ok = await membershipTable.InsertRow(newEntry, newTableVersion); Assert.IsFalse(ok, "InsertRow should have failed - same entry, old table version"); ok = await membershipTable.InsertRow(CreateMembershipEntryForTest(), newTableVersion); Assert.IsFalse(ok, "InsertRow should have failed - new entry, old table version"); data = await membershipTable.ReadAll(); var nextTableVersion = data.Version.Next(); ok = await membershipTable.InsertRow(newEntry, nextTableVersion); Assert.IsFalse(ok, "InsertRow should have failed - duplicate entry"); data = await membershipTable.ReadAll(); Assert.AreEqual(1, data.Members.Count, "only one row should have been inserted"); data = await membershipTable.ReadRow(newEntry.SiloAddress); Assert.AreEqual(newTableVersion.Version, data.Version.Version); logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", data.Version, data); Assert.AreEqual(1, data.Members.Count, "Number of records returned - data row only"); Assert.IsNotNull(data.Version.VersionEtag, "New version ETag should not be null"); Assert.AreNotEqual(newTableVersion.VersionEtag, data.Version.VersionEtag, "New VersionEtag differetnfrom last"); Assert.AreEqual(newTableVersion.Version, data.Version.Version); MembershipEntry MembershipEntry = data.Members[0].Item1; string eTag = data.Members[0].Item2; logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} Entry={1}", eTag, MembershipEntry); Assert.IsNotNull(eTag, "ETag should not be null"); Assert.IsNotNull(MembershipEntry, "MembershipEntry should not be null"); }
public async Task <MembershipTableData> ReadAll() { try { var versionEntity = await this.GetClusterVersion(); var entryEntities = await this.GetSilos(); var version = default(TableVersion); if (versionEntity != null) { version = new TableVersion(versionEntity.ClusterVersion, versionEntity.Metadata.ResourceVersion); } else { this._logger?.LogError("Initial ClusterVersionEntity entity doesn't exist."); } var memEntries = new List <Tuple <MembershipEntry, string> >(); foreach (var entity in entryEntities) { try { MembershipEntry membershipEntry = ParseEntity(entity); memEntries.Add(new Tuple <MembershipEntry, string>(membershipEntry, entity.Metadata.ResourceVersion)); } catch (Exception exc) { this._logger?.LogWarning(exc, "Failure reading all membership records from Kubernetes"); throw; } } var data = new MembershipTableData(memEntries, version); return(data); } catch (Exception exc) { this._logger?.LogWarning(exc, "Failure reading all silo entries for cluster id {clusterId}", this._clusterOptions.ClusterId); throw; } }
public async Task <MembershipTableData> ReadAll() { try { //first read just the version row so that we can check for version consistency var versionEntryKeys = new Dictionary <string, AttributeValue> { { $"{SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME}", new AttributeValue(this.clusterId) }, { $"{SiloInstanceRecord.SILO_IDENTITY_PROPERTY_NAME}", new AttributeValue(SiloInstanceRecord.TABLE_VERSION_ROW) } }; var versionRow = await this.storage.ReadSingleEntryAsync(this.options.TableName, versionEntryKeys, fields => new SiloInstanceRecord(fields)); if (versionRow == null) { throw new KeyNotFoundException("No version row found for membership table"); } var keys = new Dictionary <string, AttributeValue> { { $":{SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME}", new AttributeValue(this.clusterId) } }; var records = await this.storage.QueryAllAsync(this.options.TableName, keys, $"{SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME} = :{SiloInstanceRecord.DEPLOYMENT_ID_PROPERTY_NAME}", item => new SiloInstanceRecord(item)); if (records.Any(record => record.MembershipVersion > versionRow.MembershipVersion)) { this.logger.LogWarning((int)ErrorCode.MembershipBase, "Found an inconsistency while reading all silo entries"); //not expecting this to hit often, but if it does, should put in a limit return(await this.ReadAll()); } MembershipTableData data = Convert(records); if (this.logger.IsEnabled(LogLevel.Trace)) { this.logger.LogTrace("ReadAll Table {Table}", data.ToString()); } return(data); } catch (Exception exc) { this.logger.LogWarning( (int)ErrorCode.MembershipBase, exc, "Intermediate error reading all silo entries {TableName}.", options.TableName); throw; } }
public async Task <MembershipTableData> ReadRow(SiloAddress key) { var id = ConstructSiloEntityId(key); try { var spResponse = await this._dbClient.ExecuteStoredProcedureAsync <ReadResponse>( UriFactory.CreateStoredProcedureUri(this._options.DB, this._options.Collection, READ_SILO_SPROC), new RequestOptions { PartitionKey = new PartitionKey(this._options.ClusterId) }, this._options.ClusterId, id); ClusterVersionEntity versionEntity = spResponse.Response.ClusterVersion; List <SiloEntity> entryEntities = spResponse.Response.Silos; TableVersion version = null; if (versionEntity != null) { version = new TableVersion(versionEntity.ClusterVersion, versionEntity.ETag); } else { this._logger.LogError("Initial ClusterVersionEntity entity doesn't exist."); } var memEntries = new List <Tuple <MembershipEntry, string> >(); foreach (var entity in entryEntities) { try { MembershipEntry membershipEntry = ParseEntity(entity); memEntries.Add(new Tuple <MembershipEntry, string>(membershipEntry, entity.ETag)); } catch (Exception exc) { //TODO: Log it throw; } } var data = new MembershipTableData(memEntries, version); return(data); } catch (Exception exc) { this._logger.LogWarning($"Failure reading silo entry {id} for cluster id {this._options.ClusterId}: {exc}"); throw; } }
public async Task <MembershipTableData> ReadRow(SiloAddress key) { var siloId = GetSiloEntityId(key); var record = await _client.Get(null, Task.Factory.CancellationToken, new Key(_options.Namespace, _options.SetName, siloId)); var entries = new List <Tuple <MembershipEntry, string> >(); entries.Add(new Tuple <MembershipEntry, string>(ParseMembershipEntryRecord(record), record.generation.ToString())); TableVersion version = await ReadTableVersion(); var data = new MembershipTableData(entries, version); return(data); }
public async Task AzureMembership_ReadAll_0() { MembershipTableData data = await membership.ReadAll(); TableVersion tableVersion = data.Version; logger.Info("Membership.ReadAll returned VableVersion={0} Data={1}", tableVersion, data); Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row"); string eTag = tableVersion.VersionEtag; int ver = tableVersion.Version; Assert.IsNotNull(eTag, "ETag should not be null"); Assert.AreEqual(0, ver, "Initial tabel version should be zero"); }
private async Task MembershipTable_InsertRow(IMembershipTable membership) { MembershipEntry MembershipEntry = CreateMembershipEntryForTest(); MembershipTableData MembershipData = await membership.ReadAll(); Assert.IsNotNull(MembershipData, "Membership Data not null"); Assert.AreEqual(0, MembershipData.Members.Count, "Should be no data initially: {0}", MembershipData); bool ok = await membership.InsertRow(MembershipEntry, MembershipData.Version); Assert.IsTrue(ok, "InsertRow OK"); MembershipData = await membership.ReadAll(); Assert.AreEqual(1, MembershipData.Members.Count, "Should be one row after insert: {0}", MembershipData); }
/// <summary> /// Maps the specified source. /// </summary> /// <param name="src">The source.</param> /// <param name="dst">The DST.</param> /// <returns></returns> internal static MembershipTableData Map( List <OrleansEFMembership> src, MembershipTableData dst = null ) { var entries = src.Select(a => { return(new Tuple <MembershipEntry, string>( new MembershipEntry { SiloAddress = Map( a.Address, (int)a.Port, a.Generation ), Status = (SiloStatus)a.Status, ProxyPort = (int)a.ProxyPort, HostName = a.HostName, SiloName = a.SiloName, StartTime = a.StartTime, IAmAliveTime = a.IAmAliveTime, UpdateZone = a.UpdateZone, FaultZone = a.FaultZone, SuspectTimes = a.SuspectTimes?.Split(";").Where(b => !string.IsNullOrWhiteSpace(b) ).Select(b => { var split = b.Split("::"); return new Tuple <SiloAddress, DateTime>( SiloAddress.FromParsableString(split[0]), DateTimeOffset.FromUnixTimeMilliseconds(long.Parse(split[1])).UtcDateTime ); }).ToList(), }, a.DeploymentId )); }).ToList(); dst = new MembershipTableData( entries, new TableVersion(1, "whatever") ); return(dst); }
// Only used with MembershipTableGrain to wiat for primary to start. internal async Task WaitForTableToInit(IMembershipTable membershipTable) { var timespan = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(5); // This is a quick temporary solution to enable primary node to start fully before secondaries. // Secondary silos waits untill GrainBasedMembershipTable is created. for (int i = 0; i < 100; i++) { bool needToWait = false; try { MembershipTableData table = await membershipTable.ReadAll().WithTimeout(timespan); if (table.Members.Any(tuple => tuple.Item1.IsPrimary)) { logger.Info(ErrorCode.MembershipTableGrainInit1, "-Connected to membership table provider and also found primary silo registered in the table."); return; } logger.Info(ErrorCode.MembershipTableGrainInit2, "-Connected to membership table provider but did not find primary silo registered in the table. Going to try again for a {0}th time.", i); } catch (Exception exc) { var type = exc.GetBaseException().GetType(); if (type == typeof(TimeoutException) || type == typeof(OrleansException)) { logger.Info(ErrorCode.MembershipTableGrainInit3, "-Waiting for membership table provider to initialize. Going to sleep for {0} and re-try to reconnect.", timespan); needToWait = true; } else { logger.Info(ErrorCode.MembershipTableGrainInit4, "-Membership table provider failed to initialize. Giving up."); throw; } } if (needToWait) { await Task.Delay(timespan); } } }
internal static async Task MembershipTable_ReadRow_EmptyTable(IMembershipTable membership, SiloAddress siloAddress) { MembershipTableData data = await membership.ReadRow(siloAddress); TableVersion tableVersion = data.Version; logger.Info("Membership.ReadRow returned VableVersion={0} Data={1}", tableVersion, data); Assert.AreEqual(0, data.Members.Count, "Number of records returned - no table version row"); string eTag = tableVersion.VersionEtag; int ver = tableVersion.Version; logger.Info("Membership.ReadRow returned MembershipEntry ETag={0} TableVersion={1}", eTag, tableVersion); Assert.IsNotNull(eTag, "ETag should not be null"); Assert.AreEqual(0, ver, "Initial table version should be zero"); }
private MembershipTableData Convert(List<Tuple<SiloInstanceTableEntry, string>> entries) { try { var memEntries = new List<Tuple<MembershipEntry, string>>(); TableVersion tableVersion = null; foreach (var tuple in entries) { var tableEntry = tuple.Item1; if (tableEntry.RowKey.Equals(SiloInstanceTableEntry.TABLE_VERSION_ROW)) { tableVersion = new TableVersion(Int32.Parse(tableEntry.MembershipVersion), tuple.Item2); } else { try { MembershipEntry membershipEntry = Parse(tableEntry); memEntries.Add(new Tuple<MembershipEntry, string>(membershipEntry, tuple.Item2)); } catch (Exception exc) { logger.Error(ErrorCode.AzureTable_61, String.Format( "Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {0}. Ignoring this entry.", tableEntry), exc); } } } var data = new MembershipTableData(memEntries, tableVersion); return data; } catch (Exception exc) { logger.Error(ErrorCode.AzureTable_60, String.Format( "Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {0}.", Utils.EnumerableToString(entries, tuple => tuple.Item1.ToString())), exc); throw; } }
private MembershipTableData Convert(List<SiloInstanceRecord> entries) { try { var memEntries = new List<Tuple<MembershipEntry, string>>(); foreach (var tableEntry in entries) { try { MembershipEntry membershipEntry = Parse(tableEntry); memEntries.Add(new Tuple<MembershipEntry, string>(membershipEntry, tableEntry.ETag.ToString())); } catch (Exception exc) { logger.Error(ErrorCode.MembershipBase, $"Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {tableEntry}. Ignoring this entry.", exc); } } var data = new MembershipTableData(memEntries, _tableVersion); return data; } catch (Exception exc) { logger.Error(ErrorCode.AzureTable_60, $"Intermediate error parsing SiloInstanceTableEntry to MembershipTableData: {Utils.EnumerableToString(entries, e => e.ToString())}.", exc); throw; } }