//When a new shard is added, do the following
public void OnShardAdded(ShardInfo shard)
{
if (shard != null)
{
if (shard.Name != context.LocalShardName && _remoteShards != null && !_remoteShards.ContainsKey(shard.Name))
{
if (_clusterConfigMgr != null)
{
_clusterConfigMgr.UpdateClusterConfiguration();
ShardConfiguration sConfig = _clusterConfigMgr.GetShardConfiguration(context.LocalShardName);
if (sConfig == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn("ClusterManager.OnShardAdded() ", "The shard " + shard.Name + " does not exist in the configuration.");
}
return;
}
IShard remoteShard = new RemoteShard(new DualChannelFactory(), new ShardChannelFormatter(), context, _connectionRestoration);
remoteShard.Initialize(sConfig);
remoteShard.RegisterShardListener(Common.MiscUtil.CLUSTER_MANAGER, this);
lock (_remoteShards)
{
_remoteShards.Add(shard.Name, remoteShard);
}
}
}
}
}
public NodeToCSHeartbeatTask(NodeContext context, MembershipManager manager, IConnectionRestoration connectionRestoration, ClusterConfigurationManager clusterConfigMgr)
{
this._context = context;
this._clusterConfigMgr = clusterConfigMgr;
ShardConfiguration sConfig = null;
if (_clusterConfigMgr != null)
{
sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
if (_clusterConfigMgr.LatestConfiguration != null && _clusterConfigMgr.LatestConfiguration.Deployment != null)
{
_heartbeatInterval = 1000 * _clusterConfigMgr.LatestConfiguration.Deployment.HeartbeatInterval;
}
}
if (sConfig != null && sConfig.Servers != null)
{
_node = sConfig.Servers.GetServerNode(_context.LocalAddress.IpAddress.ToString());
}
this._membershipManager = manager;
this._connectionRestoration = connectionRestoration;
_running = false;
_startSignal = new System.Threading.ManualResetEvent(false);
}
public LocalShardCheckHeartbeatTask(IShard shard, NodeContext context, MembershipManager membershipManager, ClusterConfigurationManager clusterConfigMgr)
{
this._shard = shard;
this._context = context;
this._clusterConfigMgr = clusterConfigMgr;
_localShardHeartbeatReporter = new LocalShardHeartbeatReporting(_context.LocalAddress);
this._membershipManager = membershipManager;
this._membershipManager.HeartbeatReport = _localShardHeartbeatReporter;
ShardConfiguration sConfig = null;
if (_clusterConfigMgr != null)
{
sConfig = _clusterConfigMgr.GetShardConfiguration(context.LocalShardName);
}
if (sConfig != null && sConfig.NodeHeartbeatInterval > 0)
{
this._poolingThreshold = (sConfig.NodeHeartbeatInterval * 2) * 1000;
}
_running = false;
_startSignal = new ManualResetEvent(false);
//with the initialization of the heartbeat receiver, we start the send heart beat task.
_sendHearbeatTask = new LocalShardSendHeartbeatTask(context, shard, _membershipManager, clusterConfigMgr);
}
private void SetShardConfigs(DeploymentConfiguration configuration, out ShardConfiguration localShardConfig, out IDictionary <String, ShardConfiguration> remoteShardConfigs)
{
Dictionary <String, ShardConfiguration> remoteConfigDic = new Dictionary <String, ShardConfiguration>();
ShardConfiguration localShard = null;
foreach (Common.Configuration.ShardConfiguration shardConfig in configuration.Shards.Values)
{
if (shardConfig.Name.Equals(context.LocalShardName, StringComparison.OrdinalIgnoreCase))
{
localShard = shardConfig;
}
else
{
//localShard = shardConfig;
remoteConfigDic.Add(shardConfig.Name, shardConfig);
}
}
if (localShard == null)
{
throw new Alachisoft.NosDB.Common.Exceptions.ConfigurationException("Local Shard Configruation is missing");
}
localShardConfig = localShard;
remoteShardConfigs = remoteConfigDic;
}
/// <summary>
/// We need to ask the current primary to quit taking write ops
/// so that the left over ops can first be replicated
/// before the actual election mechanism begins.
/// </summary>
/// <returns></returns>
private bool RequestPrimaryToStopOperations()
{
ShardConfiguration sConfg = null;
bool endResult = false;
if (_clusterConfigMgr != null)
{
sConfg = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
}
if (sConfg != null)
{
MembershipChangeArgs args = new MembershipChangeArgs();
args.ChangeType = MembershipChangeArgs.MembershipChangeType.RestrictPrimary;
args.ServerName = _context.LocalAddress;
args.ElectionId = _latestMembership.ElectionId;
DatabaseMessage msg = new DatabaseMessage();
msg.Payload = args;
msg.NeedsResponse = true;
msg.OpCode = OpCode.RestrictPrimary;
msg.MessageType = MessageType.DBOperation;
ShardRequestBase <bool> request = _shard.CreateUnicastRequest <bool>(new Server(new Address(_latestMembership.Primary.Name, sConfg.Port), Status.Running), msg);
IAsyncResult result = request.BeginExecute();
endResult = request.EndExecute(result);
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
LoggerManager.Instance.ShardLogger.Info("MembershipMgr.RequestPrimaryToStopOperations()", "Requested primary to stop taking write operations. Primary response: " + endResult.ToString());
}
}
return(endResult);
}
public LocalShardSendHeartbeatTask(NodeContext nodeContext, IShard shard, MembershipManager membershipManager, ClusterConfigurationManager clusterConfigMgr)
{
this._nodeContext = nodeContext;
this._clusterConfigMgr = clusterConfigMgr;
ShardConfiguration config = _clusterConfigMgr.GetShardConfiguration(_nodeContext.LocalShardName);//_nodeContext.ConfigurationSession.GetDatabaseClusterConfiguration(_nodeContext.ClusterName).Deployment.GetShardConfiguration(_nodeContext.LocalShardName);
if (config == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled)
{
LoggerManager.Instance.ShardLogger.Error("LocalShardSendHeartTask", "shard configuration is null");
}
throw new Alachisoft.NosDB.Common.Exceptions.DatabaseException(" shard configuration is null");
}
if (config.NodeHeartbeatInterval > 0)
{
this._poolingThreshold = (config.NodeHeartbeatInterval) * 1000;
}
this._shard = shard;
this._membershipManager = membershipManager;
_running = false;
_startSignal = new System.Threading.ManualResetEvent(false);
}
public void Configure(IDependencyRegister register)
{
var section = ToolSection.Instance;
if (section == null)
{
throw new ConfigurationException("缺少 Projects.Tool 配直节信息。");
}
repositoryAssemblies = LoadAssemblies(section, "repository/assembly");
serviceAssemblies = LoadAssemblies(section, "service/assembly");
shardConfig = new ShardConfiguration(section);
metadataConfig = new MetadataConfiguration(repositoryAssemblies);
interceptorConfig = new InterceptorConfiguation(section);
repositoryConfig = new RepositoryConfiguation(repositoryAssemblies, register);
serviceConfig = new ServiceConfiguation(serviceAssemblies, register);
commandConfig = new CommandConfiguation(serviceAssemblies, register);
shardConfig.Load();
metadataConfig.Load();
interceptorConfig.Load();
repositoryConfig.Load();
serviceConfig.Load();
commandConfig.Load();
}
public override bool TryGetNodeDetail(out NodeDetail nodeDetail)
{
NodeDetail thisNode = new EndNodeDetail();
List <string> childernNaames = new List <string>();
ShardConfiguration shardConfig = (ShardConfiguration)Configuration;
foreach (ServerNode node in shardConfig.Servers.Nodes.Values)
{
if (node.Name.Equals(PathChunks[0]))
{
childernNaames.Add(node.Priority.ToString());
string[] childPathChunks = new string[this.PathChunks.Length - 1];
Array.Copy(this.PathChunks, 1, childPathChunks, 0, this.PathChunks.Length - 1);
ServerDetail s1 = new ServerDetail()
{
IPAddress = node.Name,
Priority = node.Priority
};
thisNode = new ServerValueDetail(PathChunks[0], PathType.Server, s1, childPathChunks);
}
}
if (PathChunks.Length == 1)
{
nodeDetail = thisNode;
return(true);
}
return(thisNode.TryGetNodeDetail(out nodeDetail));
}
// Receives the heartbeats from different nodes which are part of a shard and
// adds them to the report table.
public void ReceiveHeartbeat(Address source, HeartbeatInfo heartbeatInfo)
{
try
{
if (_localShardHeartbeatReporter != null)
{
// We need to verify if the node sending the heartbeat is part of the existing configuration.
// Updating the config is a costly process so we check the node in the existing config.
// If the node was freshly added in an existing (active) shard, we will not add the heartbeat to the report
// until it exists in the local node config instance.
if (_clusterConfigMgr != null)
{
ShardConfiguration sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
ServerNode sNode = null;
if (sConfig != null && sConfig.Servers != null)
{
sNode = sConfig.Servers.GetServerNode(source.IpAddress.ToString());
}
if (sNode == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShardCheckHeartbeatTask.ReceiveHeartbeat() ", "The node " + source + " is not part of the configuration.");
}
return;
}
}
bool isAnOldNode = _localShardHeartbeatReporter.AddToReport(source, heartbeatInfo);
if (!isAnOldNode)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShardCheckHeartbeatTask.ReceiveHeartbeat() ", "Node " + source.IpAddress + " added to the table for the first time. ");
}
lock (_membershipManager)
{
_membershipManager.HeartbeatReport = _localShardHeartbeatReporter;
}
_electionExecTask = new ElectionMechanismExecutionTask(_membershipManager, Activity.NodeJoining, new Server(source, Status.Running));
_electionExecTask.Start();
OnActivityCompleted();
}
}
}
catch (Exception e)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled)
{
LoggerManager.Instance.ShardLogger.Error("LocalShardCheckHeartbeatTask.ReceiveHeartbeat() ", e.ToString());
}
}
}
public ShardConfiguration GetShardConfiguration(string shardName)
{
ShardConfiguration sConfig = null;
if (LatestConfiguration != null && LatestConfiguration.Deployment != null)
{
sConfig = LatestConfiguration.Deployment.GetShard(shardName);
}
return(sConfig);
}
protected override void ProcessRecord()
{
var sConfiguration = new ShardConfiguration
{
Port = Port,
Name = Shard,
NodeHeartbeatInterval = HeartBeat,
};
var serverNodes = new Dictionary <string, ServerNode>();
var sNode = new ServerNode {
Name = Server.Split('[', ']')[0]
};
int priority = 1;
if (Server.Contains('['))
{
string priorityString = Server.Split('[', ']')[1];
int.TryParse(priorityString, out priority);
priority = priority > 3 ? 3 : priority;
}
sNode.Priority = priority < 1 ? 1 : priority;
serverNodes.Add(sNode.Name, sNode);
sConfiguration.Servers = new ServerNodes();
if (serverNodes.Count > 0)
{
sConfiguration.Servers.Nodes = serverNodes;
}
ClusterConfiguration config = new ClusterConfiguration
{
Name = Common.MiscUtil.CLUSTERED,
DisplayName = Name,
Deployment = new DeploymentConfiguration()
};
config.Deployment.AddShard(sConfiguration);
var current = new RemoteConfigurationManager();
current.CreateCluster(config, new SSPIClientAuthenticationCredential());
if (current.VerifyConfigurationCluster())
{
ConfigurationConnection.ConfigCluster = current.GetConfigurationClusterConfiguration();
ConfigurationConnection.ClusterConfiguration = current.GetDatabaseClusterConfig(true);
ConfigurationConnection.Current = current;
ConfigurationConnection.UpdateDatabaseClusterInfo();
}
SessionState.Path.SetLocation(ProviderUtil.DRIVE_ROOT + ":\\" + ConfigurationConnection.ClusterConfiguration.DisplayName);
}
public bool Initialize(ShardConfiguration configuration)
{
if (configuration == null)
{
throw new DistributorException("Remote Shard Configuration is Null");
}
_name = configuration.Name;
_shardPort = configuration.Port;
_servers = GetShardServerList(configuration);
return(true);
}
public bool Initialize(ShardConfiguration configuration)
{
if (configuration == null)
{
throw new ArgumentNullException("remote shard configuration is null");
}
this._name = configuration.Name;
this._shardPort = configuration.Port;
this._servers = GetShardServerList(configuration);
this._resolveDispute = new ResolveChannelDispute(context, this);
return(true);
}
public ShardValueDetail(string nodeName, ShardConfiguration configuration, string[] pathChunks)
{
NodeName = nodeName;
NodeType = PathType.Shard;
IsContainer = true;
IsValid = true;
Configuration = configuration;
PathChunks = pathChunks;
ChilderanTable = new PrintableTable();
ChilderanName = new List <string>();
ClusterInfo clusterInfo = ConfigurationConnection.ClusterInfo;
ShardInfo shardInfo = clusterInfo.GetShard(nodeName);
List <ServerInfo> running = new List <ServerInfo>();
if (shardInfo != null)
{
if (shardInfo.RunningNodes != null)
{
running = shardInfo.RunningNodes.Values.ToList();
}
}
ChilderanTable.AddHeader("Node IP Address", "Running Status", "Primary Status", "Priority");
foreach (ServerNode node in configuration.Servers.Nodes.Values)
{
ServerInfo serverInfo = null;
string isPrimery = ProviderUtil.NOT_PRIMARY;
string isRunning = ProviderUtil.NOT_RUNNING;
if (running != null)
{
serverInfo = running.Find(x => x.Address.ip.Equals(node.Name));
}
if (serverInfo != null)
{
if (running.Contains(serverInfo))
{
isRunning = ProviderUtil.RUNNING;
}
if (shardInfo.Primary != null && serverInfo.Equals(shardInfo.Primary))
{
isPrimery = ProviderUtil.PRIMARY;
}
}
ChilderanTable.AddRow(node.Name, isRunning, isPrimery, node.Priority.ToString());
ChilderanName.Add(node.Name);
}
}
internal int GetHighestNodePriority(IList <Address> nodes, IList <string> maxOplogAddress)
{
LoggerManager.Instance.SetThreadContext(new LoggerContext()
{
ShardName = _context.LocalShardName != null ? _context.LocalShardName : "", DatabaseName = ""
});
ShardConfiguration sConfig = null;
if (_clusterConfigMgr != null)
{
sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
}
if (sConfig == null || sConfig.Servers == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn("ElectionManager.GetHeighestNodePriority() ", "The shard (or the nodes of the shard) " + _context.LocalShardName + " does not exist in the configuration.");
}
return(0);
}
ServerNodes staticNodes = sConfig.Servers;
int maxPriority = Int32.MaxValue;
if (nodes != null)
{
foreach (Address node in nodes)
{
ServerNode serverNode = staticNodes.GetServerNode(node.IpAddress.ToString());
if (serverNode != null)
{
int thisPriority = serverNode.Priority;
if ((thisPriority < maxPriority) && maxOplogAddress.Contains(serverNode.Name))
{
maxPriority = thisPriority;
}
}
}
return(maxPriority);
}
else
{
throw new Exception("Membership is null.");
}
}
public void AddShardItem_Test()
{
ShardConfiguration shardConfiguration = new ShardConfiguration(3);
IShardKeyAlgorithm shardKeyAlgorithm = new EvenlyDistributedShardKeyAlgorithm(shardConfiguration);
InMemoryDB db = new InMemoryDB(shardConfiguration, shardKeyAlgorithm);
AddCustomer(db, 1);
AddCustomer(db, 2);
AddCustomer(db, 3);
AddCustomer(db, 4);
// 3 shards
var stats = db.GetTotalShardItems();
Assert.AreEqual(2, stats[0].ItemsCount);
Assert.AreEqual(1, stats[1].ItemsCount);
Assert.AreEqual(1, stats[2].ItemsCount);
// 4 shards
db.AddShard();
AddCustomer(db, 5);
AddCustomer(db, 6);
stats = db.GetTotalShardItems();
Assert.AreEqual(3, stats[0].ItemsCount);
Assert.AreEqual(2, stats[1].ItemsCount);
Assert.AreEqual(1, stats[2].ItemsCount);
// 5 shards
db.AddShard();
AddCustomer(db, 7);
AddCustomer(db, 8);
AddCustomer(db, 9);
AddCustomer(db, 10);
AddCustomer(db, 11);
stats = db.GetTotalShardItems();
Assert.AreEqual(4, stats[0].ItemsCount);
Assert.AreEqual(3, stats[1].ItemsCount);
Assert.AreEqual(2, stats[2].ItemsCount);
Assert.AreEqual(1, stats[3].ItemsCount);
Assert.AreEqual(1, stats[4].ItemsCount);
}
private IList <ConfigChangeParams> CheckForLocalShardConfigChange(ShardConfiguration oldConfig, ShardConfiguration latestConfig)
{
IList <ConfigChangeParams> configChanges = new List <ConfigChangeParams>();
if (oldConfig != null && latestConfig != null)
{
if (oldConfig.Servers != null && latestConfig.Servers != null)
{
IDictionary <string, ServerNode> oldNodes = oldConfig.Servers.Nodes;
if (oldNodes != null)
{
foreach (KeyValuePair <string, ServerNode> oldNode in oldNodes)
{
ServerNode latestNode = latestConfig.Servers.GetServerNode(oldNode.Value.Name);
if (latestNode != null)
{
if (oldNode.Value.Priority != latestNode.Priority)
{
configChanges.Add(new ConfigChangeParams(new Address(latestNode.Name, latestConfig.Port), ChangeType.PriorityChanged));
}
}
else
{
configChanges.Add(new ConfigChangeParams(new Address(oldNode.Value.Name, oldConfig.Port), ChangeType.NodeRemoved));
}
}
}
IDictionary <string, ServerNode> latestNodes = latestConfig.Servers.Nodes;
if (latestNodes != null)
{
foreach (KeyValuePair <string, ServerNode> newNode in latestNodes)
{
ServerNode oldNode = oldConfig.Servers.GetServerNode(newNode.Value.Name);
if (oldNode == null)
{
configChanges.Add(new ConfigChangeParams(new Address(newNode.Value.Name, latestConfig.Port), ChangeType.NodeAdded));
}
}
}
}
}
return(configChanges);
}
protected override void ProcessRecord()
{
var sConfiguration = new ShardConfiguration
{
Port = Port,
Name = Name,
NodeHeartbeatInterval = HeartBeat
};
var serverNodes = new Dictionary <string, ServerNode>();
var sNode = new ServerNode {
Name = Server.Split('[', ']')[0]
};
int priority = 1;
if (Server.Contains('['))
{
string priorityString = Server.Split('[', ']')[1];
int.TryParse(priorityString, out priority);
priority = priority > 3 ? 3 : priority;
}
sNode.Priority = priority < 1 ? 1 : priority;
serverNodes.Add(sNode.Name, sNode);
sConfiguration.Servers = new ServerNodes();
if (serverNodes.Count > 0)
{
sConfiguration.Servers.Nodes = serverNodes;
}
try
{
if (ConfigurationConnection.Current.AddShardToCluster(sConfiguration, null))
{
WriteObject("Shard Added Sucessfully");
}
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// function needs to be shifted from here.
/// this function provides a mini election feasibility check to avoid continuous triggering of the
/// election mechanism.
/// Steps:
/// 1. Does the node possess a majority of the connections.
/// 2. If yes, proceed to the next step where the primary is hunted for in the existing heartbeat report table.
/// </summary>
/// <param name="report"></param>
/// <returns></returns>
private bool AreElectionsFeasible(LocalShardHeartbeatReporting report, ShardConfiguration sConfig)
{
IList <Address> activeServers = report.GetReportTable.Keys.ToList();
if (activeServers.Count < (int)Math.Ceiling((double)sConfig.Servers.Nodes.Count / 2))
{
return(false);
}
else
{
if (report.PrimaryExists())
{
// We verify if a primary exists on any of the nodes at any given point in time and if a channel exists
// because there is a possibility that the primary is not stepping down and is not a valid primary either
// (catering for split - brain scenarios)
return(!ChannelExists(report.GetCurrentPrimary()));
}
}
return(true);
}
public void ChangeShardCount_Test()
{
ShardConfiguration shardConfiguration = new ShardConfiguration(3);
IShardKeyAlgorithm shardKeyAlgorithm = new EvenlyDistributedShardKeyAlgorithm(shardConfiguration);
InMemoryDB dataProvider = new InMemoryDB(shardConfiguration, shardKeyAlgorithm);
int shardId;
shardId = dataProvider.GetShardId("1");
Assert.AreEqual(1, shardId);
shardId = dataProvider.GetShardId("2");
Assert.AreEqual(2, shardId);
shardId = dataProvider.GetShardId("3");
Assert.AreEqual(3, shardId);
shardId = dataProvider.GetShardId("4");
Assert.AreEqual(1, shardId);
shardId = dataProvider.GetShardId("123");
Assert.AreEqual(3, shardId);
// 3 + 1 shards
dataProvider.AddShard();
shardId = dataProvider.GetShardId("1");
Assert.AreEqual(1, shardId);
shardId = dataProvider.GetShardId("2");
Assert.AreEqual(2, shardId);
shardId = dataProvider.GetShardId("3");
Assert.AreEqual(3, shardId);
shardId = dataProvider.GetShardId("4");
Assert.AreEqual(1, shardId);
shardId = dataProvider.GetShardId("123");
Assert.AreEqual(3, shardId);
shardId = dataProvider.GetShardId("5");
Assert.AreEqual(1, shardId);
shardId = dataProvider.GetShardId("6");
Assert.AreEqual(2, shardId);
shardId = dataProvider.GetShardId("7");
Assert.AreEqual(3, shardId);
shardId = dataProvider.GetShardId("8");
Assert.AreEqual(4, shardId);
}
private IList <Server> GetShardServerList(ShardConfiguration configuration)
{
if (configuration == null)
{
throw new ArgumentNullException("remote shard configuration is null");
}
IList <Server> serverList = new List <Server>();
int shardPort = configuration.Port;
if (configuration.Servers != null && configuration.Servers.Nodes != null)
{
foreach (Alachisoft.NosDB.Common.Configuration.ServerNode node in configuration.Servers.Nodes.Values)
{
Server server = new Server(new Address(node.Name, shardPort), Status.Stopped);
serverList.Add(server);
}
}
return(serverList);
}
public ShardedDb(ShardConfiguration shardConfiguration, IShardKeyAlgorithm algorithm,
IMainShardDataProvider mainShardDataProvider, IConnectionProvider connectionProvider)
{
this._shardConfiguration = shardConfiguration;
this._shardKeyAlgorithm = algorithm;
this._mainShardDataProvider = mainShardDataProvider;
this._connectionProvider = connectionProvider;
subShardMap = new Dictionary <int, V>();
_shardMap = new Dictionary <string, int>();
_shards = new List <int>();
var connection = connectionProvider.GetMainConnectionString();
mainShardDataProvider.Initialize(connection);
mainShardDataProvider.LoadShardMap(_shards, _shardMap);
if (subShardMap.Count < shardConfiguration.ShardCount)
{
SetupShard(shardConfiguration.ShardCount - subShardMap.Count);
}
}
public bool Initialize(ShardConfiguration configuration)
{
if (configuration == null)
{
return(false);
}
this.Name = configuration.Name;
IPAddress localIP = GetLocalAddress();
if (context != null)
{
context.LocalAddress = new Common.Net.Address(localIP, configuration.Port);
}
_membershipManager = new MembershipManager(this, context, _clusterConfigMgr);
lock (_mutexOnnodeRole)
{
if (NodeRole != NodeRole.Intermediate)
{
NodeRole = NodeRole.None;
}
}
this._shardPeersList = GetPeersList(configuration.Servers.Nodes.Values.ToArray(), configuration.Port);
//an old copy of the membership is required at the time when the node joins for comparison purposes.
//_oldMembership = context.ConfigurationSession.GetMembershipInfo(context.ClusterName, context.LocalShardName);
//this._shardPeersList = GetRunningPeersList(configuration.Port);
//this._shardPeersList = GetRunningPeersList(_oldMembership.Servers, configuration.Port);
this._traceProvider = new TraceProvider();
_resolveDispute = new ResolveChannelDispute(context, this);
return(true);
}
private IList <Server> GetShardServerList(ShardConfiguration configuration)
{
if (configuration == null)
{
throw new DistributorException(ErrorCodes.Distributor.SHARD_CONFIGURATION_NULL, new string[] { this.Name });
}
IList <Server> serverList = new List <Server>();
int shardPort = configuration.Port;
if (configuration.Servers == null || configuration.Servers.Nodes == null)
{
return(serverList);
}
foreach (ServerNode node in configuration.Servers.Nodes.Values)
{
Server server = new Server(new Address(node.Name, shardPort), Status.Stopped);
serverList.Add(server);
}
return(serverList);
}
public SqlMainShardDataProvider(ShardConfiguration shardConfiguration, IShardKeyAlgorithm algorithm)
{
_shardConfiguration = shardConfiguration;
_shardKeyAlgorithm = algorithm;
}
public SqlDb(ShardConfiguration shardConfiguration, IShardKeyAlgorithm algorithm) :
base(shardConfiguration, algorithm,
new SqlMainShardDataProvider(shardConfiguration, algorithm),
new SqlConnectionProvider())
{
}
// the method which runs in a thread.
public void Run()
{
LoggerManager.Instance.SetThreadContext(new LoggerContext()
{
ShardName = _context.LocalShardName != null ? _context.LocalShardName : "", DatabaseName = ""
});
_context.StatusLatch.WaitForAny(NodeStatus.Running);
_startSignal.WaitOne();
while (_running)
{
IDictionary <Address, HeartbeatInfo> reportTable = null;
try
{
ShardConfiguration sConfig = null;
if (_clusterConfigMgr != null)
{
sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
}
if (sConfig == null || sConfig.Servers == null)
{
if (LoggerManager.Instance.ShardLogger != null &&
LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn("LocalShardCheckHeartbeatTask.Run() ", "The shard (or the nodes of the shard) " +
_context.LocalShardName +
" does not exist in the configuration.");
}
return;
}
reportTable = _localShardHeartbeatReporter.GetReportTable;
if (reportTable != null && reportTable.Count > 0)
{
CheckHeartbeats(reportTable);
IList <Address> tentativeLostNodes = CheckForLostNodes(reportTable);
if (tentativeLostNodes != null && tentativeLostNodes.Count > 0)
{
// 1. These are the nodes lost.
// 2. The lost node needs to be removed from the heartbeats table and the missing heartbeats
// 3. Elections will be triggered when a node is lost.
foreach (var node in tentativeLostNodes)
{
if (_localShardHeartbeatReporter.GetReportTable.ContainsKey(node) &&
!ChannelExists(sConfig.Servers.GetServerNode(node.IpAddress.ToString())))
{
if (LoggerManager.Instance.ShardLogger != null &&
LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShard.CheckHeartbeatTask",
"did not receive heart beat from " + node);
}
//_localShardHeartbeatReporter.RemoveFromReport(node);
OnActivityTriggered(Activity.NodeLeaving, node);
lock (_membershipManager)
{
_membershipManager.HeartbeatReport =
(LocalShardHeartbeatReporting)_localShardHeartbeatReporter;
}
_electionExecTask = new ElectionMechanismExecutionTask(_membershipManager,
Activity.NodeLeaving, new Server(node, Status.Stopped));
_electionExecTask.Start();
OnActivityCompleted();
}
else
{
if (LoggerManager.Instance.ShardLogger != null &&
LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShard.CheckHeartbeatTask",
"did not receive heart beat from " + node + ", however channel is connected");
}
}
}
}
//if no primary is set, we proceed to the following steps:
if (!_shard.NodeRole.Equals(NodeRole.Intermediate) && _membershipManager.LatestMembership == null ||
( /*_membershipManager.LatestMembership != null &&*/
(_membershipManager.LatestMembership.Primary == null ||
( /*_membershipManager.LatestMembership.Primary != null &&*/
!ChannelExists(_membershipManager.LatestMembership.Primary)))))
{
//First, check if the conduction of elections is feasible.
//If yes, the actual election mechanism is triggered.
if (AreElectionsFeasible(_localShardHeartbeatReporter, sConfig))
{
lock (_membershipManager)
{
_membershipManager.HeartbeatReport =
(LocalShardHeartbeatReporting)_localShardHeartbeatReporter;
}
_electionExecTask = new ElectionMechanismExecutionTask(_membershipManager, Activity.GeneralElectionsTriggered, null);
_electionExecTask.Start();
}
//RTD: badddd logic.
//else if (_localShardHeartbeatReporter.PrimaryExists() &&
// _localShardHeartbeatReporter.GetCurrentPrimary() != null &&
// ChannelExists(_localShardHeartbeatReporter.GetCurrentPrimary()))
//{
// ServerNode currentPrimaNode = _localShardHeartbeatReporter.GetCurrentPrimary();
// if (currentPrimaNode != null)
// {
// if (LoggerManager.Instance.ShardLogger != null &&
// LoggerManager.Instance.ShardLogger.IsDebugEnabled)
// LoggerManager.Instance.ShardLogger.Debug("LocalShard.CheckHeartbeatTask",
// "no primary set so far, setting " + currentPrimaNode.Name +
// " as primary as other nodes are reporting");
// MembershipChangeArgs args = new MembershipChangeArgs();
// args.ChangeType = MembershipChangeArgs.MembershipChangeType.PrimarySet;
// args.ElectionId = _localShardHeartbeatReporter.GetCurrentElectionId();
// args.ServerName = new Address(currentPrimaNode.Name, sConfig.Port);
// ((LocalShard)_shard).OnMembershipChanged(args);
// }
//}
}
LogMembership();
}
}
catch (ThreadAbortException e)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled && _checkHbThread != null)
{
LoggerManager.Instance.ShardLogger.Error(_checkHbThread.Name, "Task aborted.");
}
break;
}
catch (Exception e)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled)
{
LoggerManager.Instance.ShardLogger.Error("LocalShardCheckHeartbeatTask.Run() ", e.ToString());
}
}
lock (_syncMutex)
{
Monitor.Wait(_syncMutex, _poolingThreshold);
}
_startSignal.WaitOne();
}
}
internal ElectionVote CastVote(Election election, Membership existingMembership, HeartbeatInfo info)
{
if (election == null || election.ElectionId == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn("ElectionManager.CastVote()", "Invalid election.");
}
return(null);
}
ElectionVote vote = new ElectionVote();
vote.RequestingNode = election.ElectionId.RequestingNode;
vote.Sourcenode = new ServerNode();
vote.Sourcenode.Name = _context.LocalAddress.IpAddress.ToString();
vote.Sourcenode.Priority = (int)LocalServerPriority;
if (existingMembership != null && (election.ElectionType == ElectionType.GeneralElections && existingMembership.Primary == null) || election.ElectionType == ElectionType.TakeoverElections)
{
if (election.RequestingServerInfo.LastOperationId > LocalServerInfo().LastOperationId)
{
vote.NodeVote = ElectionVote.Vote.yes;
}
else if (election.RequestingServerInfo.LastOperationId == LocalServerInfo().LastOperationId)
{
if (election.ElectionId.RequestingNode.Priority <= (int)LocalServerPriority)
{
vote.NodeVote = ElectionVote.Vote.yes;
}
else
{
ShardConfiguration sConfig = null;
if (_clusterConfigMgr != null)
{
sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
}
int configuredNodes = 0;
if (sConfig != null || sConfig.Servers != null)
{
configuredNodes = sConfig.Servers.Nodes.Count;
}
if (info != null && info.CSStatus == ConnectivityStatus.Connected && (configuredNodes != null && existingMembership.Servers.Count >= configuredNodes / 2))
{
vote.NodeVote = ElectionVote.Vote.no;
}
else
{
vote.NodeVote = ElectionVote.Vote.yes;
}
}
}
else
{
vote.NodeVote = ElectionVote.Vote.no;
}
}
else
{
vote.NodeVote = ElectionVote.Vote.no;
}
return(vote);
}
public void Run()
{
LoggerManager.Instance.SetThreadContext(new LoggerContext()
{
ShardName = _context.LocalShardName != null ? _context.LocalShardName : "", DatabaseName = ""
});
_context.StatusLatch.WaitForAny(NodeStatus.Running);
_startSignal.WaitOne();
if (_context.ConfigurationSession != null)
{
((OutProcConfigurationSession)_context.ConfigurationSession).RegisterListener(this);
}
while (_running)
{
try
{
if (_node == null)
{
_clusterConfigMgr.UpdateClusterConfiguration();
}
if (_clusterConfigMgr != null)
{
ShardConfiguration sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
if (sConfig != null && sConfig.Servers != null)
{
_node = sConfig.Servers.GetServerNode(_context.LocalAddress.IpAddress.ToString());
}
}
if (_node == null && LoggerManager.Instance.ShardLogger != null &&
LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn("NodeToCSHeartbeatTask.Run() ", "Node " + _context.LocalAddress.ToString() +
" is not part of the configuration.");
return;
}
OperationId lastOpId = null;
if (_membershipManager != null && _membershipManager.LatestMembership != null && _membershipManager.LatestMembership.Primary != null && _membershipManager.LatestMembership.Primary.Name.Equals(_context.LocalAddress.IpAddress.ToString()))
{
lastOpId = _membershipManager.GetLastOperationId;
}
Stopwatch watch = new Stopwatch();
watch.Start();
_heartbeatInterval =
_context.ConfigurationSession.ReportHeartbeat(_context.ClusterName, _context.LocalShardName,
_node, _membershipManager.LatestMembership, lastOpId) * 1000;
watch.Stop();
_csStatus = ConnectivityStatus.Connected;
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("NodeToCSHeartbeatTask.Run() ", "Heartbeat sent to the CS at " +
DateTime.Now.ToString() + " time taken to report heartbeat :" + watch.Elapsed.TotalSeconds);
}
if (_heartbeatInterval > 0 && (watch.Elapsed.TotalSeconds > _heartbeatInterval / 2))
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled)
{
LoggerManager.Instance.ShardLogger.Error("NodeToCSHeartbeatTask.Run() ", "Heartbeat sent to the CS at " +
DateTime.Now.ToString() + " time taken to report heartbeat :" + watch.Elapsed.TotalSeconds + " which is greater than half of the hb interval.");
}
}
}
catch (ThreadAbortException e)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled && _nodeToCSThread != null)
{
LoggerManager.Instance.ShardLogger.Error(_nodeToCSThread.Name, "Task aborted.");
}
break;
}
//the following should only be done when a connection exception occurs.
catch (ChannelException e)
{
if (LoggerManager.Instance.ShardLogger != null)
{
if (LoggerManager.Instance.ShardLogger.IsErrorEnabled)
{
LoggerManager.Instance.ShardLogger.Error("NodeToCSHeartbeatTask.Run() ", e.ToString());
}
if (LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("NodeToCSHeartbeatTask.Run() ",
"On CS disconnected process of the membership manager begins execution at " +
DateTime.Now.ToString());
}
}
_csStatus = ConnectivityStatus.NotConnected;
_membershipManager.OnCSDisconnected();
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("NodeToCSHeartbeatTask.Run() ", "The NodeToCS task paused at " +
DateTime.Now.ToString());
}
//foreach (var cluster in ManagementServer.s_DbmNodeConfiguration.DbmClusters.ClustersConfigurations)
//{
// string csIp = cluster.ConfigServers.Nodes[0].Name;
// int csPort = cluster.ConfigServers.Nodes[0].Port;
// BrokenConnectionInfo info = new BrokenConnectionInfo();
// info.BrokenAddress = new Address(csIp, csPort);
// info.SessionType = Common.Communication.SessionTypes.Management;
// _connectionRestoration.RegisterListener(info, this,_context.LocalShardName);
//}
this.Pause();
}
catch (Exception e)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled)
{
LoggerManager.Instance.ShardLogger.Error("NodeToCSHbTask.Run() General Exception: ",
e.ToString());
}
if (e.Message.Contains("No configuration server is available to process the request"))
{
_csStatus = ConnectivityStatus.NotConnected;
_membershipManager.OnCSDisconnected();
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("NodeToCSHeartbeatTask.Run() ", "The NodeToCS task paused at " +
DateTime.Now.ToString());
}
this.Pause();
}
}
Stopwatch sleepWatch = new Stopwatch();
sleepWatch.Start();
Thread.Sleep(_heartbeatInterval);
_startSignal.WaitOne();
sleepWatch.Stop();
if (sleepWatch.Elapsed.TotalMilliseconds > (_heartbeatInterval + 2000))
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("NodeToCSHeartbeatTask.Run() ", "CS task waited for " + sleepWatch.Elapsed.TotalSeconds);
}
}
}
}
public void TriggerElectionMechanism(Activity activity, Server server, LocalShardHeartbeatReporting heartbeatReport, Membership existingMembership)
{
LoggerManager.Instance.SetThreadContext(new LoggerContext()
{
ShardName = _context.LocalShardName != null ? _context.LocalShardName : "", DatabaseName = ""
});
if (existingMembership == null)
{
existingMembership = new Membership();
}
ShardConfiguration sConfig = null;
//Get the shard configuration
if (_clusterConfigMgr != null)
{
sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
}
IList <Address> activeNodes = null;
MembershipChangeArgs args = new MembershipChangeArgs();
ServerNodes staticServerNodes = null;
if (sConfig == null || sConfig.Servers == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn("ElectionBasedMembershipStrategy.TiggerElectionMechanism() ", "The shard " + _context.LocalShardName + " does not exist in the configuration.");
}
return;
}
staticServerNodes = sConfig.Servers;
ElectionResult result = null;
if (heartbeatReport != null)
{
activeNodes = heartbeatReport.GetReportTable.Keys.ToList();
}
Address activityNode = null;
if (server == null)
{
activityNode = _context.LocalAddress;
}
else
{
activityNode = server.Address;
}
switch (activity)
{
case Activity.NodeJoining:
if (server == null)
{
return;
}
//On node join, we need to get membership from the config server for the first time.
Membership csMembership = _context.ConfigurationSession.GetMembershipInfo(_context.ClusterName, _context.LocalShardName);
ServerNode joiningNode = sConfig.Servers.GetServerNode(server.Address.IpAddress.ToString());
// If the added node is configured while the cluster is up and running, do the following.
if (joiningNode == null)
{
if (_clusterConfigMgr != null)
{
_clusterConfigMgr.UpdateClusterConfiguration();
sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
}
if (sConfig == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn("ElectionBasedMembershipStrategy.TriggerElectionMechanism() ", "The shard " + _context.LocalShardName + " does not exist in the configuration.");
}
return;
}
joiningNode = sConfig.Servers.GetServerNode(server.Address.IpAddress.ToString());
}
if (joiningNode == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn("ElectionBasedMembershipStrategy.TriggerElectionMechanism() ", "The node " + server.Address + " is not part of the configuration.");
}
return;
}
if (existingMembership == null || existingMembership.Servers == null || !existingMembership.Servers.Contains(joiningNode))
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("ElectBasedMemSt.TriggerElectMech", "Node joining activity triggered for " + activityNode);
}
}
bool thisNodeIsPrimary = false;
OperationId lastOpId = null;
if (heartbeatReport != null && heartbeatReport.GetReportTable.ContainsKey(server.Address))
{
args.ServerName = _context.LocalAddress;
args.ElectionId = null;
args.ChangeType = MembershipChangeArgs.MembershipChangeType.NodeJoined;
if (server.Address.Equals(_context.LocalAddress))
{
_context.ConfigurationSession.ReportNodeJoining(_context.ClusterName, _context.LocalShardName, sConfig.Servers.GetServerNode(_context.LocalAddress.IpAddress.ToString()));
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
LoggerManager.Instance.ShardLogger.Info("electBasedMemSt.TriggerElectMech", server.Address + " reported its joining to the config server. ");
}
//if the primary is not null and the channel is not disconnected, it can be set here.
if ((existingMembership == null || existingMembership.Primary == null) && csMembership.Primary != null && _shard.ActiveChannelsList.Contains(new Server(new Address(csMembership.Primary.Name, sConfig.Port), Status.Initializing)) && ObeysMajorityRule(_shard.ActiveChannelsList.Count, sConfig.Servers.Nodes.Count))
{
args.ServerName = new Address(csMembership.Primary.Name, sConfig.Port);
args.ElectionId = csMembership.ElectionId;
args.ChangeType = MembershipChangeArgs.MembershipChangeType.PrimarySet;
//if the node which was lost comes back up before the CS or the nodes can declare it dead,
//it should resume its status as a primary. There should be no need for an election in this case.
if (args.ServerName.Equals(_context.LocalAddress))
{
thisNodeIsPrimary = true;
}
}
}
if (thisNodeIsPrimary)
{
if (csMembership.ElectionId != null && LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
LoggerManager.Instance.ShardLogger.Info("electBasedMemSt.TriggerElectMech", "election_id: " + csMembership.ElectionId.Id + " election time :" + csMembership.ElectionId.ElectionTime);
}
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
LoggerManager.Instance.ShardLogger.Info("electBasedMemSt.TriggerElectMech", "I am already declared primary");
}
lastOpId = LastOperationId;
ChangeMembershipShardwide(args);
}
else
{
((LocalShard)_shard).OnMembershipChanged(args);
}
if (server.Address.Equals(_context.LocalAddress))
{
ServerNode sNode = sConfig.Servers.GetServerNode(_context.LocalAddress.IpAddress.ToString());
if (sNode == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn("ElectionBasedMembershipStrategy.TriggerElectionMechanism() ", "The node " + sNode.Name + " does not exist in the configuration.");
}
return;
}
_context.ConfigurationSession.ReportHeartbeat(_context.ClusterName, _context.LocalShardName, sNode, existingMembership, lastOpId);
}
}
else
{
if (existingMembership.Primary != null && existingMembership.Primary.Name.Equals(server.Address.IpAddress.ToString()))
{
if (sConfig.Servers == null || sConfig.Servers.Nodes == null || !ObeysMajorityRule(activeNodes.Count, sConfig.Servers.Nodes.Count))
{
_context.ConfigurationSession.SetNodeStatus(_context.ClusterName, _context.LocalShardName, existingMembership.Primary, NodeRole.None);
args.ChangeType = MembershipChangeArgs.MembershipChangeType.PrimaryDemoted;
args.ServerName = _context.LocalAddress;
ChangeMembershipShardwide(args);
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
LoggerManager.Instance.ShardLogger.Info("electBasedMemSt.TriggerElectMech", " Node addition activity occured. Primary node " + _context.LocalAddress.IpAddress.ToString() + " demoted.");
}
return;
}
}
}
break;
case Activity.NodeLeaving:
if (server == null)
{
return;
}
bool hasMajority = ObeysMajorityRule(activeNodes.Count, staticServerNodes.Nodes.Count);
args.ServerName = server.Address;
args.ChangeType = MembershipChangeArgs.MembershipChangeType.NodeLeft;
_clusterConfigMgr.UpdateClusterConfiguration();
if (existingMembership.Primary != null)
{
// if the existing primary is actually the node lost, we need to update the configuration.
if (existingMembership.Primary.Name == server.Address.IpAddress.ToString())
{
//if Primary leaves, it should be updated locally.
args.ChangeType = MembershipChangeArgs.MembershipChangeType.PrimaryLost;
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.TriggerElectMech", "Node leaving activity triggered for " + server.Address + " . Primary lost.");
}
}
else if (existingMembership.Primary.Name == _context.LocalAddress.IpAddress.ToString()) // if the existing primary is the local node, we need to check for possible demotion of the current primary.
{
if (!hasMajority)
{
_context.ConfigurationSession.SetNodeStatus(_context.ClusterName, _context.LocalShardName, existingMembership.Primary, NodeRole.None);
args.ChangeType = MembershipChangeArgs.MembershipChangeType.PrimaryDemoted;
args.ServerName = _context.LocalAddress;
ChangeMembershipShardwide(args);
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.TriggerElectMech", " Node leaving activity occurred. Primary node " + _context.LocalAddress.IpAddress.ToString() + " demoted.");
}
return;
}
}
}
((LocalShard)_shard).OnMembershipChanged(args);
break;
case Activity.GeneralElectionsTriggered:
case Activity.TakeoverElectionsTriggered:
// this is where the actual election mechanism takes place.
//Step 1: if no node in the heartbeat table has a primary and there is no primary in the local node's membership, we proceed forward.
//Else if there is a primary but this looks like the takeover election mechanism, we proceed along as well.
if ((activity.Equals(Activity.GeneralElectionsTriggered) && !heartbeatReport.PrimaryExists() && existingMembership.Primary == null) || (activity.Equals(Activity.TakeoverElectionsTriggered) && heartbeatReport.PrimaryExists()))
{
//Step 2: we verify that this node has a majority of the shard nodes connected to it.
if (activeNodes != null && ObeysMajorityRule(activeNodes.Count, staticServerNodes.Nodes.Count))
{
//Step 3: Perform the initial sanity check. (Speculative phase)
if (ShouldIInitiateElection(heartbeatReport, activity))
{
if (existingMembership != null && existingMembership.Primary != null && activity == Activity.GeneralElectionsTriggered)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
LoggerManager.Instance.ShardLogger.Info("electBasedMemSt.TriggerElectMech", "A primary has already been selected for "
+ _context.LocalShardName + " hence exiting the election mechanism.");
}
return;
}
//Step 4: The elections take place in real. (Authoritative Phase)
result = HoldElection(heartbeatReport, activity);
if (result != null)
{
if (result.PollingResult == ElectionResult.Result.PrimarySelected)
{
//if the shard is undergoing the takeover election mechanism, the old primary needs to
//be demoted first.
bool oldPrimaryDemoted = false;
if (activity == Activity.TakeoverElectionsTriggered)
{
MembershipChangeArgs args2 = new MembershipChangeArgs();
args2.ChangeType = MembershipChangeArgs.MembershipChangeType.ForcefullyDemotePrimary;
args2.ServerName = _context.LocalAddress;
args2.ElectionId = existingMembership.ElectionId;
Message msg = new Message();
msg.Payload = args2;
msg.MessageType = MessageType.MembershipOperation;
msg.NeedsResponse = true;
ShardRequestBase <bool> request = _shard.CreateUnicastRequest <bool>(new Server(new Address(existingMembership.Primary.Name, sConfig.Port), Status.Running), msg);
IAsyncResult result2 = request.BeginExecute();
oldPrimaryDemoted = request.EndExecute(result2);
}
//Submit the result to the CS.
if (activity == Activity.GeneralElectionsTriggered || (activity == Activity.TakeoverElectionsTriggered && oldPrimaryDemoted))
{
_context.ConfigurationSession.SubmitElectionResult(_context.ClusterName.ToLower(), _context.LocalShardName.ToLower(), result);
}
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.TriggerElectMech", "Election result submitted for shard " + _context.LocalShardName.ToString());
}
_context.ElectionResult = result;
args.ServerName = _context.LocalAddress;
args.ElectionId = result.ElectionId;
args.ChangeType = MembershipChangeArgs.MembershipChangeType.PrimarySelected;
//Once, the result is submitted, inform the shard nodes.
ChangeMembershipShardwide(args);
_context.ConfigurationSession.ReportHeartbeat(_context.ClusterName, _context.LocalShardName, result.ElectedPrimary, existingMembership, LastOperationId);
}
//Finally, end this round of elections.
_context.ConfigurationSession.EndElection(_context.ClusterName, _context.LocalShardName, result.ElectionId);
}
}
}
}
break;
case Activity.CSDisconnected:
//this is called whenever a node loses connection with the config server.
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("ElectionBasedMembershipStrategy.TriggerElectionMechanism() ", "Config Server disconnected. ");
}
//if the number of configured nodes are even and the primary loses connection with the CS, it needs to demote itself.
if (existingMembership != null && existingMembership.Primary != null && existingMembership.Primary.Name == _context.LocalAddress.IpAddress.ToString() && staticServerNodes.Nodes.Count % 2 == 0)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.TriggerElectMech", " Connection of the node " + _context.LocalAddress.ToString() + " with the config server is lost.");
}
args.ServerName = _context.LocalAddress;
args.ElectionId = existingMembership.ElectionId;
args.ChangeType = MembershipChangeArgs.MembershipChangeType.PrimaryDemoted;
ChangeMembershipShardwide(args);
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
LoggerManager.Instance.ShardLogger.Info("electBasedMemSt.TriggerElectMech", " Primary node " + _context.LocalAddress.IpAddress.ToString() + " demoted because the primary lost connection with the CS.");
}
}
break;
case Activity.ForcefulPrimaryDemotion:
if (existingMembership != null && existingMembership.Primary != null && existingMembership.Primary.Name == _context.LocalAddress.IpAddress.ToString())
{
_context.ConfigurationSession.SetNodeStatus(_context.ClusterName, _context.LocalShardName, existingMembership.Primary, NodeRole.None);
args.ChangeType = MembershipChangeArgs.MembershipChangeType.PrimaryDemoted;
args.ServerName = _context.LocalAddress;
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
LoggerManager.Instance.ShardLogger.Info("electBasedMemSt.TriggerElectMech", "Primary node " + _context.LocalAddress.IpAddress.ToString() + " demoted in order to complete the take over election mechanism. ");
}
((LocalShard)_shard).OnMembershipChanged(args);
}
break;
}
}