private long _weightBalanceThreshold = 0 ; //at what weight should the node be treated as contributor to incoming nodes.
public DistributionMatrix(ArrayList weightIdList, Address address, DistributionData distData, ILogger NCacheLog)
{
_address = address;
_distData = distData;
_filteredWeightIdList = new ArrayList();
_itemsCount = weightIdList.Count;
_totalWeight = 1;
_weightToSacrifice = 0;
_cushionFactor = 10;
_percentWeightToSacrifice = 0;
_weightBalanceThreshold = Convert.ToInt32((_maxCacheSize * WeightBalanceThresholdPercent) / 100); //10%, threshold at which we feel to balance weight for incoming nodes. its value is percent of MaxCacheSize
if (NCacheLog.IsInfoEnabled) NCacheLog.Error("DistributionMatrix.ctor", "Address->" + address.ToString() + ", DistributionData->" + distData.ToString());
//muds:
//this is the temp code just to put some trace...
int bucketCount = 0;
foreach (WeightIdPair wiPair in weightIdList)
{
if (wiPair.Address.compare(address) == 0)
{
if(NCacheLog.IsInfoEnabled) NCacheLog.Info("DistributionMatrix.ctor", "waitPair" + wiPair.Address.ToString() + ", wiPait->" + wiPair.BucketId);
_filteredWeightIdList.Add(wiPair);
bucketCount++;
}
}
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("DistributionMatrix..ctor", address + " owns " + bucketCount + " buckets");
_filteredWeightIdList.Sort();
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("DistributionMatrix.ctor", "_filterWeightIdList.Count:" + _filteredWeightIdList.Count + ", distData.BucketPerNode: " + distData.BucketsPerNode);
//Current bucket count - bucketss count after division gives buckets count to be sacrificed.
_bucketsToSacrifice = _filteredWeightIdList.Count - distData.BucketsPerNode;
if (_bucketsToSacrifice <= 0)
{
NCacheLog.Error("DistributionMatrix", "Address::" + address.ToString() + " cant sacrifice any bucket. Buckets/Node = " + distData.BucketsPerNode + " My Buckets Count = " + _filteredWeightIdList.Count);
return;
}
int rows = Convert.ToInt32(Math.Ceiling((double)((decimal)_filteredWeightIdList.Count /(decimal)_bucketsToSacrifice)));
int cols = _bucketsToSacrifice;
InitializeMatrix(rows, cols);
}
internal virtual void notifyConnectionClosed(Address peer)
{
NCacheLog.CriticalInfo("ConnectionTable.notifyConnectionClosed", peer.ToString() + " connection close notification");
if (peer == null)
return;
for (int i = 0; i < conn_listeners.Count; i++)
((ConnectionTable.ConnectionListener)conn_listeners[i]).connectionClosed(peer);
}
/// <summary>Try to obtain correct Connection (or create one if not yet existent) </summary>
protected virtual Connection getConnection(Address dest, Address primaryAddress, bool reEstablishCon, bool isPrimary, bool withFirstNIC,bool connectingFirstTime)
{
Connection conn = null;
System.Net.Sockets.Socket sock;
Address peer_addr = null;
try
{
if (primaryAddress == null) primaryAddress = dest;
if (withFirstNIC)
{
if (isPrimary)
conn = (Connection)conns_NIC_1[dest];
else
conn = (Connection)secondayrConns_NIC_1[dest];
}
else
{
if (isPrimary)
conn = (Connection)conns_NIC_2[primaryAddress];
else
conn = (Connection)secondayrConns_NIC_2[primaryAddress];
}
if ((conn == null ||!connectingFirstTime) && reEstablishCon)
{
try
{
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.getConnection()", "No Connetion was found found with " + dest.ToString());
if (local_addr == null) return null; //cluster being stopped.
sock = Connect(dest, withFirstNIC);
conn = new Connection(this, sock, primaryAddress, this.NCacheLog, isPrimary, nagglingSize, _retries, _retryInterval);
conn.MemManager = MemManager;
conn.IamInitiater = true;
ConnectInfo conInfo = null;
try
{
byte connectStatus = connectingFirstTime ? ConnectInfo.CONNECT_FIRST_TIME : ConnectInfo.RECONNECTING;
conn.sendLocalAddress(local_addr,connectingFirstTime);
conn.readPeerAddress(sock,ref peer_addr);
if (((Address)local_addr).CompareTo((Address)dest) > 0)
{
conInfo = new ConnectInfo(connectStatus, GetConnectionId());
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.getConnection", dest + " I should send connect_info");
conn.SendConnectInfo(conInfo);
}
else
{
conInfo = conn.ReadConnectInfo(sock);
}
//log.Error("ConnectionTable.getConnection", " conn_info :" + conInfo);
conn.ConInfo = conInfo;
}
catch (System.Exception e)
{
NCacheLog.Error("ConnectionTable.getConnection()", e.Message);
conn.DestroySilent();
return null;
}
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.getConnection", "b4 lock conns.SyncRoot");
try
{
conn_syn_lock.AcquireWriterLock(Timeout.Infinite);
if (isPrimary)
{
if (withFirstNIC)
{
if (conns_NIC_1.ContainsKey(dest))
{
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.getConnection()", "connection is already in the table");
Connection tmpConn = (Connection)conns_NIC_1[dest];
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.getConnection", "table_con id :" + tmpConn.ConInfo.Id + " new_con id :" + conn.ConInfo.Id);
if (conn.ConInfo.Id < tmpConn.ConInfo.Id)
{
conn.Destroy();
return tmpConn;
}
else
{
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.getConnection()", dest + "--->connection present in the table is terminated");
tmpConn.Destroy();
conns_NIC_1.Remove(dest);
}
}
notifyConnectionOpened(dest);
}
else
{
if (conns_NIC_2.ContainsKey(primaryAddress))
{
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.getConnection()", "connection is already in the table");
Connection tmpConn = (Connection)conns_NIC_1[dest];
if (conn.ConInfo.Id < tmpConn.ConInfo.Id)
{
conn.Destroy();
return tmpConn;
}
else
{
NCacheLog.Warn("ConnectionTable.getConnection()", dest + "connection present in the table is terminated");
tmpConn.Destroy();
conns_NIC_2.Remove(primaryAddress);
}
}
}
}
addConnection(primaryAddress, conn, isPrimary, withFirstNIC);
if (useDedicatedSender) AddDedicatedMessageSender(primaryAddress, conn, withFirstNIC);
conn.init();
}
finally
{
conn_syn_lock.ReleaseWriterLock();
}
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.getConnection", "after lock conns.SyncRoot");
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.getConnection()", "connection is now working");
}
finally
{
}
}
}
catch (System.Threading.ThreadAbortException e)
{
if (conn != null) conn.Destroy();
conn = null;
}
catch (System.Threading.ThreadInterruptedException ex)
{
if (conn != null) conn.Destroy();
conn = null;
}
finally
{
}
return conn;
}
public void ReceivedHeartBeat(Address sender, HearBeat hrtBeat)
{
Message rspMsg;
switch (hrtBeat.Type)
{
case HearBeat.HEART_BEAT:
if (_enclosingInstance.Stack.NCacheLog.IsInfoEnabled) _enclosingInstance.Stack.NCacheLog.Info("ConnectionKeepAlive.ReceivedHeartBeat", "received heartbeat from ->:" + sender.ToString());
lock (_idleConnections.SyncRoot)
{
_idleConnections.Remove(sender);
}
break;
case HearBeat.SEND_HEART_BEAT:
rspMsg = new Message(sender, null, new byte[0]);
rspMsg.putHeader(HeaderType.KEEP_ALIVE, new HearBeat(HearBeat.HEART_BEAT));
if (_enclosingInstance.Stack.NCacheLog.IsInfoEnabled) _enclosingInstance.Stack.NCacheLog.Info("ConnectionKeepAlive.ReceivedHeartBeat", "seding heartbeat to ->:" + sender.ToString());
_enclosingInstance.sendUnicastMessage(rspMsg, false, rspMsg.Payload, Priority.Critical);
break;
case HearBeat.ARE_YOU_ALIVE:
rspMsg = new Message(sender, null, new byte[0]);
HearBeat rsphrtBeat = (_enclosingInstance.isClosing || _enclosingInstance._leaving) ? new HearBeat(HearBeat.I_AM_LEAVING) : new HearBeat(HearBeat.I_AM_NOT_DEAD);
rsphrtBeat = _enclosingInstance.isStarting ? new HearBeat(HearBeat.I_AM_STARTING) : rsphrtBeat;
rspMsg.putHeader(HeaderType.KEEP_ALIVE, rsphrtBeat);
if (_enclosingInstance.Stack.NCacheLog.IsInfoEnabled) _enclosingInstance.Stack.NCacheLog.Info("ConnectionKeepAlive.ReceivedHeartBeat", "seding status" + rsphrtBeat + " to ->:" + sender.ToString());
_enclosingInstance.sendUnicastMessage(rspMsg, false, rspMsg.Payload, Priority.Critical);
break;
case HearBeat.I_AM_STARTING:
case HearBeat.I_AM_LEAVING:
case HearBeat.I_AM_NOT_DEAD:
lock (_status_mutex)
{
if (_currentSuspect != null && _currentSuspect.Equals(sender))
{
_statusReceived = hrtBeat;
Monitor.Pulse(_status_mutex);
}
}
break;
}
}
public Connection Reconnect(Address node)
{
Connection peerConnection = null;
bool shouldConnect = false;
bool initiateReconnection = false;
if (node == null)
{
NCacheLog.Error("ConnectionTable.Reconnect", "node name is NULL");
return null;
}
lock (con_reestablish_sync)
{
try
{
if (_nodeRejoiningList != null)
{
lock (_nodeRejoiningList.SyncRoot)
{
int localNodeIndex = -1;
int nodeIndex = -1;
for (int i = 0; i < _nodeRejoiningList.Count; i++)
{
Address listNode = _nodeRejoiningList[i] as Address;
if (listNode.Equals(node))
{
nodeIndex = i;
}
if (listNode.Equals(LocalAddress))
{
localNodeIndex = i;
}
}
if (nodeIndex >= 0 && localNodeIndex >= 0)
{
shouldConnect = true;
if (nodeIndex > localNodeIndex)
{
initiateReconnection = true;
}
}
}
}
if (shouldConnect)
{
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.Reconnect", node.ToString() + " is part of node rejoining list");
int connectionRetries = _retries;
while (connectionRetries-- > 0)
{
peerConnection = ReEstablishConnection(node);
if (peerConnection == null)
{
Thread.Sleep(new TimeSpan(0, 0, _retryInterval));
}
else
{
break;
}
}
if (peerConnection == null)
{
if (NCacheLog.IsErrorEnabled) NCacheLog.Error("ConnectionTable.Reconnect", "Can not establish connection with " + node + " after " + _retries + " retries");
notifyConnectionClosed(node);
}
else
{
NCacheLog.CriticalInfo("ConnectionTable.Reconnect", "Connection re-establised with " + node);
if (peerConnection.IamInitiater)
{
//inform above layers about re-connection.
System.Threading.ThreadPool.QueueUserWorkItem(new WaitCallback(InformAboutReconnection), node);
}
}
}
else
{
if (NCacheLog.IsInfoEnabled) NCacheLog.Info("ConnectionTable.Reconnect", node.ToString() + " is not part of the node rejoining list");
notifyConnectionClosed(node);
}
}
catch (Exception e)
{
NCacheLog.Error("ConnectionTable.Reconnect", "An error occurred while reconnecting with " + node + " Error :" + e.ToString());
notifyConnectionClosed(node);
}
}
return peerConnection;
}
/// <summary> Resend the mcast request with the given localSeqID
///
/// </summary>
/// <param name="seqID">the local sequence id of the
/// </param>
private void _retransmitMcastRequest(long seqID, Address groupSequencerAddr)
{
// *** Get a shared lock
try
{
stateLock.AcquireReaderLock(Timeout.Infinite); try
{
if (Stack.NCacheLog.IsInfoEnabled) Stack.NCacheLog.Info("TOTAL._retransmitMcastRequest()", "Retransmit MCAST_REQ[" + seqID + "] to : " + groupSequencerAddr.ToString());
_transmitMcastRequest(seqID, groupSequencerAddr);
// ** Revoke the shared lock
}
finally
{
stateLock.ReleaseReaderLock();
}
}
catch (ThreadInterruptedException ex)
{
Stack.NCacheLog.Error("Protocols.TOTAL._retransmitMCastRequest", ex.ToString());
}
}
public override void handleInformNodeRejoining(Address sender, Address node)
{
if (node != null)
{
if (gms.Stack.NCacheLog.IsInfoEnabled) gms.Stack.NCacheLog.Info("CoordinatorGmsImpl.handleInformNodeRejoining", sender.ToString() + " informed about rejoining with " + node);
if (gms.members.contains(node))
{
ViewId viewId = gms.GetNextViewId();
GMS.HDR header = new GMS.HDR(GMS.HDR.RESET_ON_NODE_REJOINING, node);
header.view = new View(viewId, gms.members.Clone() as ArrayList);
header.view.CoordinatorGmsId = gms.unique_id;
Message rejoiningMsg = new Message(null, null, new byte[0]);
rejoiningMsg.putHeader(HeaderType.GMS, header);
gms.passDown(new Event(Event.MSG, rejoiningMsg, Priority.Critical));
}
}
}
private void handleSignalEndOfStateTxfr(Address requestingNode)
{
if (_corresponders != null)
{
StateTxfrCorresponder cor = _corresponders[requestingNode] as StateTxfrCorresponder;
if (cor != null)
{
if (NCacheLog.IsInfoEnabled)
NCacheLog.Info("PartitionedServerCache.handleSignalEndOfTxfr",
requestingNode.ToString() + " Corresponder removed.");
cor.Dispose();
_corresponders.Remove(requestingNode);
}
}
}
/// <summary>
/// Transfers the buckets from a its owner. We may receive data in chunks.
/// It is a pull model, a node wanting state transfer a bucket makes request
/// to its owner.
/// </summary>
/// <param name="buckets"></param>
/// <param name="owner"></param>
/// <returns></returns>
private bool TransferBuckets(ArrayList buckets, ref Address owner, bool sparsedBuckets)
{
bool transferEnd;
bool successfullyTxfrd = false;
int expectedTxfrId = 1;
bool resync = false;
try
{
if (_parent.Context.NCacheLog.IsInfoEnabled) _parent.Context.NCacheLog.Info(Name + ".TransferBuckets", " Starting transfer. Owner : " + owner.ToString() + " , Bucket : " + ((int)buckets[0]).ToString());
PrepareBucketsForStateTxfr(buckets);
long dataRecieved = 0;
long currentIternationData = 0;
while (true)
{
if (_enableGc && _dataTransferred >= _gcThreshhold)
{
_dataTransferred = 0;
DateTime start = DateTime.Now;
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
DateTime end = DateTime.Now;
TimeSpan diff = end - start;
if (_parent.NCacheLog.IsErrorEnabled) _parent.NCacheLog.CriticalInfo(this.Name + ".TransferBucket", "explicit GC called. time taken(ms) :" + diff.TotalMilliseconds + " gcThreshold :" + _gcThreshhold);
}
else
_dataTransferred += currentIternationData;
Boolean sleep = false;
resync = false;
transferEnd = true;
StateTxfrInfo info = null;
try
{
currentIternationData = 0;
info = SafeTransferBucket(buckets, owner, sparsedBuckets, expectedTxfrId);
if (info != null)
{
currentIternationData = info.DataSize;
dataRecieved += info.DataSize;
}
}
catch (Runtime.Exceptions.SuspectedException)
{
resync = true;
}
catch (Runtime.Exceptions.TimeoutException)
{
}
if (resync)
{
if (_parent.Context.NCacheLog.IsInfoEnabled) _parent.Context.NCacheLog.Info(Name + ".TransferBuckets", owner + " is suspected");
Address changedOwner = GetChangedOwner((int)buckets[0], owner);
if (changedOwner != null)
{
if (_parent.Context.NCacheLog.IsInfoEnabled) _parent.Context.NCacheLog.Info(Name + ".TransferBuckets", changedOwner + " is new owner");
if (changedOwner.Equals(owner))
{
continue;
}
else
{
owner = changedOwner;
expectedTxfrId = 1;
continue;
}
}
else
{
_parent.Context.NCacheLog.Error(Name + ".TransferBuckets", " Could not get new owner");
info = new StateTxfrInfo(true);
}
}
if (info != null)
{
successfullyTxfrd = true;
transferEnd = info.transferCompleted;
HashVector tbl = info.data;
CacheEntry entry = null;
expectedTxfrId++;
//muds:
//add data to local cache.
if (tbl != null && tbl.Count > 0)
{
IDictionaryEnumerator ide = tbl.GetEnumerator();
while (ide.MoveNext())
{
if (!_stateTransferEventLogged)
{
AppUtil.LogEvent(_cacheserver, "\"" + _parent.Context.SerializationContext + "(" + _parent.Cluster.LocalAddress.ToString() + ")\"" + " has started state transfer.", System.Diagnostics.EventLogEntryType.Information, EventCategories.Information, EventID.StateTransferStart);
_stateTransferEventLogged = _logStateTransferEvent = true;
}
try
{
OperationContext operationContext = new OperationContext(OperationContextFieldName.OperationType, OperationContextOperationType.CacheOperation);
if (ide.Value != null)
{
entry = ide.Value as CacheEntry;
CacheInsResultWithEntry result = _parent.InternalCache.Insert(ide.Key, entry, false, false, null, LockAccessType.DEFAULT, operationContext);
if (result != null && result.Result == CacheInsResult.NeedsEviction)
{
failedKeysList.Add(ide.Key);
}
}
else
{
_parent.InternalCache.Remove(ide.Key, ItemRemoveReason.Removed, false, false, null, LockAccessType.IGNORE_LOCK, operationContext);
}
}
catch (StateTransferException se)
{
_parent.Context.NCacheLog.Error(Name + ".TransferBuckets", " Can not add/remove key = " + ide.Key + " : value is " + ((ide.Value == null) ? "null" : " not null") + " : " + se.Message);
}
catch (Exception e)
{
_parent.Context.NCacheLog.Error(Name + ".TransferBuckets", " Can not add/remove key = " + ide.Key + " : value is " + ((ide.Value == null) ? "null" : " not null") + " : " + e.Message);
}
}
if (_parent.Context.NCacheLog.IsInfoEnabled) _parent.Context.NCacheLog.Info(Name + ".TransferBuckets", " BalanceDataLoad = " + _isBalanceDataLoad.ToString());
if (_isBalanceDataLoad)
_parent.Context.PerfStatsColl.IncrementDataBalPerSecStatsBy(tbl.Count);
else
_parent.Context.PerfStatsColl.IncrementStateTxfrPerSecStatsBy(tbl.Count);
}
}
else
successfullyTxfrd = false;
if (transferEnd)
{
BucketsTransfered(owner, buckets);
EndBucketsStateTxfr(buckets);
//muds:
//send ack for the state transfer over.
//Ask every node to release lock on this/these bucket(s)
if (_parent.Context.NCacheLog.IsInfoEnabled) _parent.Context.NCacheLog.Info(Name + ".TransferBuckets", "Acknowledging transfer. Owner : " + owner.ToString() + " , Bucket : " + ((int)buckets[0]).ToString());
AcknowledgeStateTransferCompleted(owner, buckets);
break;
}
if (info != null)
_throttlingManager.Throttle(info.DataSize);
}
}
catch (System.Threading.ThreadAbortException)
{
EndBucketsStateTxfr(buckets);
throw;
}
return successfullyTxfrd;
}
internal bool allowJoin(Address mbr, bool isStartedAsMirror)
{
if (!isPartReplica) return true;
//muds:
//new code for disabling the join while in state transfer.
lock (acquirePermission_mutex)
{
_allowJoin = false;
ArrayList existingMembers = members.Members;
Address lastJoiney = null;
if (existingMembers.Count > 0)
{
lastJoiney = existingMembers[existingMembers.Count - 1] as Address;
if (isStartedAsMirror)
{
if (!lastJoiney.IpAddress.Equals(mbr.IpAddress))
return false;
}
else
{
if (existingMembers.Count > 1)
{
Address secondLastJoinee = existingMembers[existingMembers.Count - 2] as Address;
if (!lastJoiney.IpAddress.Equals(secondLastJoinee.IpAddress))
return false;
}
else
return false;
}
}
if (isStartedAsMirror)
{
//if members contain my active then i need to join
if (members.ContainsIP(mbr))
{
_allowJoin = true;
}
}
else
{
Stack.NCacheLog.CriticalInfo("pbcast.GMS.allowJoin()", "Join permission for " + mbr.ToString());
bool inStateTransfer = IsClusterInStateTransfer();
_allowJoin = !inStateTransfer;
}
return _allowJoin;
}
}