public static ArrayList DistributeOrphanBuckets(ArrayList hashMap, Address leavingNode, ArrayList members, out Hashtable orphanedBuckets)
{
orphanedBuckets = new Hashtable();
HashMapBucket tempBuck;
hashMap = ReAllocBucketsInTransfer(hashMap, leavingNode);
int[] bucketsInEachNode = NodeBucketsCount(hashMap, members); //node vs bucket count.
bool bAssigned = false;
int memberCount = members.Count;
if (memberCount == 0)
{
return(null);
}
int bucketsPerNode = hashMap.Count / members.Count;
for (int i = 0, j = 0; i < hashMap.Count; i++)
{
j = (j == memberCount) ? 0 : j;
tempBuck = (HashMapBucket)hashMap[i];
if (tempBuck.PermanentAddress.CompareTo(leavingNode) == 0)
{
bAssigned = false;
for (int k = 0; k < memberCount; k++)
{
if (bucketsInEachNode[j] < bucketsPerNode)
{
HashMapBucket clonedBucket = tempBuck.Clone() as HashMapBucket;
orphanedBuckets.Add(clonedBucket.BucketId, clonedBucket);
Address mbr = members[j] as Address;
bucketsInEachNode[j] = bucketsInEachNode[j] + 1; //increment bucket count as next j is incremented.
tempBuck.PermanentAddress = mbr;
tempBuck.TempAddress = mbr;
tempBuck.Status = BucketStatus.Functional;
j++;
bAssigned = true;
break;
}
else
{
j++;
j = (j == memberCount) ? 0 : j;
}
}
//exceptional case when last node gets few more buckets. Assign those leftover buckets to ANY node.
if (bAssigned == false)
{
Address mbr = (Address)members[j++];;
tempBuck.PermanentAddress = tempBuck.TempAddress = mbr;
tempBuck.Status = BucketStatus.Functional;
}
}
}
return(hashMap);
}
/// <summary>
/// Verifies whether a given node is the permanent owner of the bucket or not
/// </summary>
/// <param name="bucketId"></param>
/// <param name="perOwner"></param>
/// <returns></returns>
public bool VerifyPermanentOwnership(int bucketId, Address perOwner)
{
try
{
if (perOwner != null)
{
Sync.AcquireReaderLock(Timeout.Infinite);
try
{
lock (InstalledHashMap.SyncRoot)
{
HashMapBucket bucket = (HashMapBucket)InstalledHashMap[bucketId];
return(perOwner.Equals(bucket.TempAddress));
}
}
finally
{
Sync.ReleaseReaderLock();
}
}
}
catch (Exception e)
{
NCacheLog.Error("DistributionMgr.VerifyPermanentOwnership", e.ToString());
}
return(false);
}
/// <summary>
/// Returns a table that has buckets owners and there respective hashcode ranges
/// </summary>
/// <returns>a hashtable, with bucket id's as keys and owners address as value</returns>
internal Hashtable GetOwnerHashMapTable(Hashtable renderers)
{
Hashtable ownerHashCodeTable = new Hashtable(TotalBuckets);
try
{
_sync.AcquireReaderLock(Timeout.Infinite);
if (_installedHashMap != null)
{
for (int i = 0; i < _installedHashMap.Count; i++)
{
HashMapBucket bucket = _installedHashMap[i] as HashMapBucket;
if (bucket != null)
{
switch (bucket.Status)
{
case BucketStatus.Functional:
ownerHashCodeTable.Add(i, GetServerAddress(renderers, bucket.PermanentAddress));
break;
case BucketStatus.NeedTransfer:
case BucketStatus.UnderStateTxfr:
ownerHashCodeTable.Add(i, GetServerAddress(renderers, bucket.TempAddress));
break;
}
}
}
}
}
finally
{
_sync.ReleaseReaderLock();
}
return(ownerHashCodeTable);
}
public virtual void WaitForBucketStatus(int bucket, byte status, Address owner)
{
if (owner != null)
{
while (true)
{
HashMapBucket hashBucket = GetBucketForWait(bucket, owner);
if (hashBucket == null)
{
return;
}
if (hashBucket.StateTxfrLatch.IsAnyBitsSet(status) || !owner.Equals(hashBucket.TempAddress))
{
return;
}
lock (_status_wait_mutex)
{
if (!owner.Equals(hashBucket.TempAddress) || hashBucket.StateTxfrLatch.IsAnyBitsSet(status))
{
return;
}
Monitor.Wait(_status_wait_mutex);
}
}
}
}
public NodeBalanceData(ArrayList hashMap, Hashtable bucketStatistics, ArrayList members)
{
_percentWeightPerNode = 100 / members.Count;
_members = members;
int memberCount = _members.Count;
_hashMapData = new ArrayList(memberCount);
_cacheDataSum = 1;
ArrayList _weightIdList = new ArrayList();
for (int i = 0; i < DistributionManager.TotalBuckets; i++)
{
HashMapBucket hmapBuck = (HashMapBucket)hashMap[i];
BucketStatistics buckStats = (BucketStatistics)bucketStatistics[i];
if (hmapBuck.Status != BucketStatus.UnderStateTxfr) //include only those buckets that are Functional
{
WeightIdPair listItem = new WeightIdPair(hmapBuck.BucketId, buckStats.DataSize, hmapBuck.PermanentAddress);
_weightIdList.Add(listItem);
}
_cacheDataSum += buckStats.DataSize; //Lets get the TOTAL weight of the cluster.
}
_weightPerNode = _cacheDataSum / memberCount;
_balanceDataListForNodes = new ArrayList(memberCount);
foreach (Address mbr in _members)
{
BalanceDataForNode balanceData = new BalanceDataForNode(_weightIdList, mbr, _cacheDataSum);
_balanceDataListForNodes.Add(balanceData);
}
}
public bool IsBucketFunctional(Address owner, object key)
{
int bucketId = GetBucketId(key as string);
Sync.AcquireReaderLock(Timeout.Infinite);
try
{
if (BucketsOwnershipMap != null && BucketsOwnershipMap.Contains(owner))
{
ArrayList buckets = BucketsOwnershipMap[owner] as ArrayList;
if (buckets != null)
{
int index = buckets.IndexOf(new HashMapBucket(null, bucketId));
if (index != -1)
{
HashMapBucket bucket = buckets[index] as HashMapBucket;
if (bucket.Status == BucketStatus.Functional)
{
return(true);
}
}
}
}
}
finally
{
Sync.ReleaseReaderLock();
}
return(false);
}
public void Set(int bucketId, string group)
{
if (_installedHashMap != null)
{
HashMapBucket bucket = (HashMapBucket)_installedHashMap[bucketId];
bucket.StateTxfrLatch.SetStatusBit(BucketStatus.Functional, BucketStatus.UnderStateTxfr);
}
}
public void SetBucketStatus(HashMapBucket bucket, byte status)
{
if (_bucket.BucketId == bucket.BucketId)
{
_bucket.Status = status;
_bucket.CurrentShard = bucket.FinalShard;
}
}
public void UpdateBucketStats(NodeInfo localNode)
{
try
{
Sync.AcquireWriterLock(Timeout.Infinite);
if (localNode == null)
{
return;
}
if (_bucketsStats == null)
{
_bucketsStats = new Hashtable();
}
if (localNode.Statistics != null && localNode.Statistics.LocalBuckets != null)
{
Hashtable bucketStats = localNode.Statistics.LocalBuckets.Clone() as Hashtable;
if (bucketStats != null)
{
IDictionaryEnumerator ide = bucketStats.GetEnumerator();
while (ide.MoveNext())
{
//muds:
//see if this node is the permanent owner of the bucket
//otherwise its quite possible that we override the
//stats of the bucket from the temporary owner.
HashMapBucket bucket = (HashMapBucket)_installedHashMap[(int)ide.Key];
if (bucket.PermanentAddress.Equals(localNode.Address))
{
BucketStatistics stats = ide.Value as BucketStatistics;
_bucketsStats[ide.Key] = ide.Value;
}
else
{
}
}
}
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("DistributionMgr.UpdateBucketStats()", "bucketStats = " + _bucketsStats == null ? "null" : _bucketsStats.Count.ToString());
}
}
}
catch (Exception e)
{
if (NCacheLog.IsErrorEnabled)
{
NCacheLog.Error("DistributionMgr.UpdateBucketStats()", e.ToString());
}
}
finally
{
Sync.ReleaseWriterLock();
}
}
/// <summary>
/// Locks the buckets which are under the process of state transfer. A locked
/// bucket can not be assigned to a node while loadbalancing. Only a coordinator
/// node can lock the buckets.
/// </summary>
/// <param name="buckets"></param>
/// <param name="requestingNode"></param>
/// <returns></returns>
public virtual Hashtable LockBuckets(ArrayList buckets, Address requestingNode)
{
ArrayList lockAcquired = new ArrayList();
ArrayList ownerChanged = new ArrayList();
Hashtable result = new Hashtable();
Sync.AcquireWriterLock(Timeout.Infinite);
try
{
if (buckets != null)
{
IEnumerator ie = buckets.GetEnumerator();
while (ie.MoveNext())
{
lock (InstalledHashMap.SyncRoot)
{
HashMapBucket bucket = (HashMapBucket)InstalledHashMap[(int)ie.Current];
if (requestingNode.Equals(bucket.TempAddress))
{
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("DistributionMgr.lockbuckets", "acquired locked on bucket [" + bucket.BucketId + "] by " + requestingNode);
}
bucket.Status = BucketStatus.UnderStateTxfr;
if (!lockAcquired.Contains(ie.Current))
{
lockAcquired.Add(ie.Current);
}
}
else if (!ownerChanged.Contains(ie.Current))
{
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("DistributionMgr.lockbuckets", "bucket [" + bucket.BucketId + "] owner ship is changed; new owner is " + bucket.TempAddress);
}
ownerChanged.Add(ie.Current);
}
}
}
}
}
catch (Exception e)
{
NCacheLog.Error("DistributionMgr.lockbuckets", e.ToString());
return(result);
}
finally
{
Sync.ReleaseWriterLock();
}
result[BucketLockResult.OwnerChanged] = ownerChanged;
result[BucketLockResult.LockAcquired] = lockAcquired;
return(result);
}
protected virtual void WaitForBucketStatus(int bucket, byte status)
{
HashMapBucket bkt = _installedHashMap[bucket] as HashMapBucket;
if (bkt != null)
{
bkt.StateTxfrLatch.WaitForAny(status);
}
}
/// <summary>
///
/// </summary>
/// <param name="newMap"></param>
/// <param name="newBucketsOwnershipMap"></param>
/// <param name="leftMbrs"></param>
public void InstallHashMap(DistributionMaps distributionMaps, ArrayList leftMbrs)
{
ArrayList newMap = null;
Hashtable newBucketsOwnershipMap = null;
_sync.AcquireWriterLock(Timeout.Infinite);
try
{
if (distributionMaps == null)
{
return;
}
newMap = distributionMaps.Hashmap;
newBucketsOwnershipMap = distributionMaps.BucketsOwnershipMap;
if (newMap == null || newBucketsOwnershipMap == null)
{
return;
}
if (_installedHashMap != null)
{
for (int i = 0; i < newMap.Count; i++)
{
HashMapBucket newBucket = (HashMapBucket)newMap[i];
int index = _installedHashMap.IndexOf(newBucket);
HashMapBucket oldBucket = (HashMapBucket)_installedHashMap[index];
if (!oldBucket.PermanentAddress.Equals(newBucket.PermanentAddress) && oldBucket.TempAddress.Equals(newBucket.TempAddress))
{
NCacheLog.Error("Install Hasmap", "BucketID: " + index.ToString() + "\toldBucket: " + oldBucket.PermanentAddress.ToString() + "\toldBucket.Temp: " + oldBucket.TempAddress.ToString() + "\tnewBucket: " + newBucket.PermanentAddress.ToString() + "\tnewBucekt.Temp: " + newBucket.TempAddress.ToString());
}
else
{
oldBucket.PermanentAddress = newBucket.PermanentAddress;
oldBucket.TempAddress = newBucket.TempAddress;
oldBucket.Status = newBucket.Status;
}
}
}
else
{
_installedHashMap = newMap;
}
_bucketsOwnershipMap = newBucketsOwnershipMap;
NotifyBucketUpdate();
}
finally
{
_sync.ReleaseWriterLock();
}
}
public DistributionData(ArrayList hashMap, Hashtable bucketStatistics, ArrayList members, ILogger NCacheLog, long cacheSizePerNode)
{
_members = members;
int memberCount = _members.Count;
_hashMapData = new ArrayList(memberCount);
_cacheDataSum = 1;
ArrayList _weightIdList = new ArrayList();
for (int i = 0; i < DistributionManager.TotalBuckets; i++)
{
HashMapBucket hmapBuck = (HashMapBucket)hashMap[i];
BucketStatistics buckStats = (BucketStatistics)bucketStatistics[i];
if (buckStats == null)
{
buckStats = new BucketStatistics();
}
//Catering for situations when two nodes are balancing and a new node joins in OR
// two nodes joins one after the other, first one started state transfer while second jumped in.
if (hmapBuck.Status != BucketStatus.UnderStateTxfr) //include only those buckets that are Functional/NeedStateTr
{
//We are selecting buckets based on temp address; although it is possible that these buckets
//might have not been transfered to TEMP owner but algorithm consider these are owned by TEMP owner.
WeightIdPair listItem = new WeightIdPair(hmapBuck.BucketId, buckStats.DataSize, hmapBuck.TempAddress);
_weightIdList.Add(listItem);
}
_cacheDataSum += buckStats.DataSize; //Lets get the TOTAL weight of the cluster.
}
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("DistributionData()", "cacheDataSum = " + _cacheDataSum.ToString());
}
//Initialize the two very important data pieces. All distribution is based on this.
_bucketsPerNode = DistributionManager.TotalBuckets / (memberCount + 1);
_weightPerNode = _cacheDataSum / (memberCount + 1);
_distMatrixForNodes = new ArrayList(memberCount);
long maxCacheSize = cacheSizePerNode * memberCount; //in bytes..CacheSize/node is the one user has entered while creating the cluster
foreach (Address mbr in _members)
{
DistributionMatrix distMatrix = new DistributionMatrix(_weightIdList, mbr, this, NCacheLog);
distMatrix.MaxCacheSize = maxCacheSize;
_distMatrixForNodes.Add(distMatrix);
}
}
//While a node leaves, all those buckets that were in transfer of buckets to the leaving node, should sieze transfer and would
// clear up tempAdd to be the same as the perm. addr.
private static ArrayList ReAllocBucketsInTransfer(ArrayList hashMap, Address leavingNode)
{
for (int i = 0; i < hashMap.Count; i++)
{
HashMapBucket bucket = (HashMapBucket)hashMap[i];
if (bucket.TempAddress.CompareTo(leavingNode) == 0)
{
bucket.TempAddress = ((HashMapBucket)hashMap[i]).PermanentAddress;
bucket.Status = BucketStatus.Functional;
}
}
return(hashMap);
}
/// <summary>
/// Set the status of the bucket to state transfer and in this way this
/// bucket becomes locked. A locked bucket can not be assigned during
/// loadbalancing.
/// </summary>
/// <param name="buckets"></param>
/// <param name="node"></param>
public void ChangeBucketStatusToStateTransfer(ArrayList buckets, Address node)
{
Sync.AcquireWriterLock(Timeout.Infinite);
try
{
if (buckets != null)
{
IEnumerator ie = buckets.GetEnumerator();
while (ie.MoveNext())
{
lock (InstalledHashMap.SyncRoot)
{
HashMapBucket bucket = (HashMapBucket)InstalledHashMap[(int)ie.Current];
if (node.Equals(bucket.TempAddress))
{
bucket.Status = BucketStatus.UnderStateTxfr;
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("DistributionMgr.ChangeBucketStatus", bucket.ToString());
}
}
}
}
if (_bucketsOwnershipMap != null)
{
ArrayList nodeBuckets = _bucketsOwnershipMap[node] as ArrayList;
if (nodeBuckets != null)
{
foreach (int bucketId in buckets)
{
int indexOfBucket = nodeBuckets.IndexOf(new HashMapBucket(null, bucketId));
if (indexOfBucket != -1)
{
HashMapBucket bucket = nodeBuckets[indexOfBucket] as HashMapBucket;
if (node.Equals(bucket.TempAddress))
{
bucket.Status = BucketStatus.UnderStateTxfr;
}
}
}
}
}
}
}
finally
{
Sync.ReleaseWriterLock();
}
}
public int GetBucketId(string key)
{
int hashCode = AppUtil.GetHashCode(key);
int index = hashCode / this.BucketSize;
if (index < 0)
{
index = index * -1;
}
HashMapBucket bucket = _installedHashMap[index] as HashMapBucket;
return(bucket.BucketId);
}
private Address SelectNodeInternal(string key)
{
int hashCode = AppUtil.GetHashCode(key);
int index = hashCode / this.BucketSize;
if (index < 0)
{
index = index * -1;
}
_sync.AcquireReaderLock(Timeout.Infinite);
try
{
if (_installedHashMap != null)
{
HashMapBucket bucket = _installedHashMap[index] as HashMapBucket;
if (bucket != null)
{
//if (bucket.Status != BucketStatus.Functional)
{
/// This is special case that handles operations during stateTransfer.
/// If a bucket is not yet transfered to the new coordinator from the replica. then
/// the replica's address is returned.
Address coordinatorNodeAddress = bucket.TempAddress; // this should be the sub-coordinator addres
ArrayList ownershipMap = _bucketsOwnershipMap[coordinatorNodeAddress] as ArrayList;
if (ownershipMap == null)
{
NCacheLog.Warn("DistributionManager.SelectNodeInternal()", "ownershipMap is null. Returning permanent address of bucket.");
return(bucket.PermanentAddress);
}
int indexOfOwnedBucket = ownershipMap.IndexOf(bucket);
if (indexOfOwnedBucket != -1)
{
HashMapBucket ownedBucket = ownershipMap[indexOfOwnedBucket] as HashMapBucket;
return(ownedBucket.PermanentAddress);
}
}
return(bucket.PermanentAddress);
}
}
return(null);
}
finally
{
_sync.ReleaseReaderLock();
}
}
protected void RemoveBucketKeyIndex(DocumentKey key)
{
HashMapBucket bucket = _parent.GetKeyBucket(key);
if (bucket != null)
{
lock (_bucketKeyIndex)
{
if (_bucketKeyIndex.ContainsKey(bucket.BucketId))
{
_bucketKeyIndex[bucket.BucketId].Remove(key);
}
}
}
}
public virtual void Wait(object key)
{
if (key != null)
{
if (_installedHashMap != null)
{
int bucketId = GetBucketId(key as string);
HashMapBucket bucket = (HashMapBucket)_installedHashMap[bucketId];
bucket.StateTxfrLatch.WaitForAny(BucketStatus.Functional | BucketStatus.NeedTransfer);
}
else
{
NCacheLog.Error("DistributionManager.Wait", "_installedHashMap == null");
}
}
}
private void ApplyChangesInHashMap(BalanceDataForNode secNode)
{
ArrayList weightIdPair = secNode.WeightIdList;
Address newAddr = secNode.NodeAddress;
HashMapBucket bucket = null;
foreach (WeightIdPair widPair in weightIdPair)
{
bucket = (HashMapBucket)_hashMap[widPair.BucketId];
if (!newAddr.Equals(bucket.TempAddress))
{
bucket.Status = BucketStatus.NeedTransfer;
}
bucket.TempAddress = newAddr;
}
}
protected void AddBucketKeyIndex(DocumentKey key)
{
HashMapBucket bucket = _parent.GetKeyBucket(key);
if (bucket != null)
{
lock (_bucketKeyIndex)
{
if (!_bucketKeyIndex.ContainsKey(bucket.BucketId))
{
_bucketKeyIndex.Add(bucket.BucketId, new ClusteredList <DocumentKey>());
}
_bucketKeyIndex[bucket.BucketId].Add(key);
}
}
}
/// <summary>
///
/// </summary>
/// <returns></returns>
public virtual Hashtable GetBucketsOwnershipMap(ArrayList hashMap)
{
Hashtable bucketsOwnershipMap = new Hashtable();
_sync.AcquireReaderLock(Timeout.Infinite);
try
{
if (hashMap != null)
{
IEnumerator ie = hashMap.GetEnumerator();
while (ie.MoveNext())
{
HashMapBucket bucket = ie.Current as HashMapBucket;
Address owner = bucket.TempAddress != null ? bucket.TempAddress : bucket.PermanentAddress;
if (bucketsOwnershipMap.Contains(owner))
{
ArrayList buckets = bucketsOwnershipMap[owner] as ArrayList;
if (!buckets.Contains(bucket))
{
buckets.Add(bucket.Clone());
}
}
else
{
ArrayList buckets = new ArrayList();
buckets.Add(bucket.Clone());
bucketsOwnershipMap[owner] = buckets;
}
}
}
}
finally
{
_sync.ReleaseReaderLock();
}
return(bucketsOwnershipMap);
}
public override DistributionMaps GetMaps(DistributionInfoData distInfoData)
{
ArrayList tmpMap = null;
Hashtable bucketsOwnershipMap = null;
ArrayList partitionNodes = new ArrayList();
if (_installedHashMap == null)
{
tmpMap = new ArrayList(TotalBuckets);
for (int i = 0; i < TotalBuckets; i++)
{
HashMapBucket bucket = new HashMapBucket(distInfoData.AffectedNode.NodeAddress, i, BucketStatus.Functional);
tmpMap.Add(bucket);
}
_existingMembers.Add(distInfoData.AffectedNode.NodeAddress);
_subGroupMap[distInfoData.AffectedNode.NodeAddress] = distInfoData.AffectedNode.SubGroup;
//for each new group we are keeping list of members. For only Partition it will be one ..for POR can be greater then one.
//This is new member, the first one. So create the list here.
distInfoData.AffectedNode.IsCoordinator = true;
partitionNodes.Add(distInfoData.AffectedNode);
_partitionNodesInfo.Add(distInfoData.AffectedNode.SubGroup, partitionNodes); //A hash table keeping list of addresses against each GROUP/Partition.
_lastCreatedHashMap = tmpMap.Clone() as ArrayList;
bucketsOwnershipMap = GetBucketsOwnershipMap(_lastCreatedHashMap);
return(new DistributionMaps(_lastCreatedHashMap, bucketsOwnershipMap));
}
//for non-coordinator node that recently becomes coordinator...
else if (_lastCreatedHashMap == null)
{
_lastCreatedHashMap = _installedHashMap.Clone() as ArrayList;
}
switch (distInfoData.ClustActivity)
{
case ClusterActivity.NodeJoin:
try
{
//assuming existing members doesnot contain the newly added member.
if (!_partitionNodesInfo.ContainsKey(distInfoData.AffectedNode.SubGroup))
{
partitionNodes = new ArrayList();
distInfoData.AffectedNode.IsCoordinator = true;
partitionNodes.Add(distInfoData.AffectedNode);
_subGroupMap[distInfoData.AffectedNode.NodeAddress] = distInfoData.AffectedNode.SubGroup;
_partitionNodesInfo.Add(distInfoData.AffectedNode.SubGroup, partitionNodes); //A hash table keeping list of addresses against each GROUP/Partition.
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("DistributionMgr.GetMaps()", "Sending new map as a new node joined the cluster");
}
return(GetMapsOnNodeJoining(distInfoData));
}
else
{
partitionNodes = (ArrayList)_partitionNodesInfo[distInfoData.AffectedNode.SubGroup];
partitionNodes.Add(distInfoData.AffectedNode);
_subGroupMap[distInfoData.AffectedNode.NodeAddress] = distInfoData.AffectedNode.SubGroup;
return(new DistributionMaps(_lastCreatedHashMap, GetBucketsOwnershipMap(_lastCreatedHashMap)));
}
}
catch (Exception e)
{
if (NCacheLog.IsErrorEnabled)
{
NCacheLog.Error("DistributionMgr.GetMaps()", e.ToString());
}
break;
}
case ClusterActivity.NodeLeave:
//assuming existing members do not containt the node to be removed/left.
_existingMembers.Remove(distInfoData.AffectedNode.NodeAddress);
_subGroupMap.Remove(distInfoData.AffectedNode.NodeAddress);
//Check if this node is the only one in partition or not.So better do distribution
if (IsLastNodeInPartition(distInfoData.AffectedNode))
{
_partitionNodesInfo.Remove(distInfoData.AffectedNode.SubGroup);
return(GetMapsOnNodeLeaving(distInfoData));
}
else //this mean we still have nodes available for this partition.
{
ArrayList groupNodes = (ArrayList)_partitionNodesInfo[distInfoData.AffectedNode.SubGroup];
if (IsCoordinatorNodeInPartition(distInfoData.AffectedNode))
{
groupNodes.Remove((object)distInfoData.AffectedNode);
((PartNodeInfo)groupNodes[0]).IsCoordinator = true;
_partitionNodesInfo[distInfoData.AffectedNode.SubGroup] = groupNodes;
_existingMembers.Add(((PartNodeInfo)groupNodes[0]).NodeAddress);
tmpMap = UpgradeToCoordinatorOfReplica(distInfoData.AffectedNode.NodeAddress, ((PartNodeInfo)groupNodes[0]).NodeAddress);
_lastCreatedHashMap = tmpMap.Clone() as ArrayList;
bucketsOwnershipMap = GetBucketsOwnershipMap(_lastCreatedHashMap);
return(new DistributionMaps(_lastCreatedHashMap, bucketsOwnershipMap));
}
else
{
//simply remove the node and get a new bucket ownership map.
groupNodes.Remove(distInfoData.AffectedNode);
return(new DistributionMaps(_lastCreatedHashMap, GetBucketsOwnershipMap(_lastCreatedHashMap)));
}
}
case ClusterActivity.None:
BalanceNodeMgr bnMgr = new BalanceNodeMgr(null);
DistributionMaps result = bnMgr.BalanceNodes(distInfoData, _lastCreatedHashMap, _bucketsStats, _existingMembers);
if (result.Hashmap != null)
{
_lastCreatedHashMap = result.Hashmap.Clone() as ArrayList;
result.BucketsOwnershipMap = GetBucketsOwnershipMap(_lastCreatedHashMap);
}
return(result);
default:
break;
}
return(null);
}
public void Deserialize(Common.Serialization.IO.CompactReader reader)
{
Name = reader.ReadObject() as string;
_distributionSequence = reader.ReadInt32();
_bucket = reader.ReadObject() as HashMapBucket;
}
public NonShardedDistribution(string shard)
{
_bucket = new HashMapBucket(shard, 0, BucketStatus.Functional);
_distributionSequence = 0;
}
/// <summary>
/// Updates the state transfer task in synchronus way. It adds/remove buckets
/// to be transferred by the state transfer task.
/// </summary>
/// <param name="myBuckets"></param>
public override bool UpdateStateTransfer(ArrayList myBuckets, int updateId)
{
if (_databasesManager != null && _databasesManager.HasDisposed(taskIdentity.DBName, taskIdentity.ColName) /* _parent.HasDisposed*/)
{
return(false);
}
StringBuilder sb = new StringBuilder();
lock (_updateIdMutex)
{
if (updateId != updateCount)
{
if (LoggerManager.Instance.StateXferLogger != null && LoggerManager.Instance.StateXferLogger.IsInfoEnabled)
{
LoggerManager.Instance.StateXferLogger.Info(loggingModule + "UpdateStateTxfr", " Do not need to update the task as update id does not match; provided id :" + updateId + " currentId :" + updateCount);
}
return(false);
}
}
lock (_stateTxfrMutex)
{
try
{
if (myBuckets != null)
{
if (LoggerManager.Instance.StateXferLogger != null && LoggerManager.Instance.StateXferLogger.IsInfoEnabled)
{
LoggerManager.Instance.StateXferLogger.Info(loggingModule + ".UpdateStateTxfr", " my buckets " + myBuckets.Count);
}
//we work on the copy of the map.
ArrayList buckets = myBuckets.Clone() as ArrayList;
ArrayList leavingShards = new ArrayList();
//if (_sparsedBuckets != null && _sparsedBuckets.Count > 0)
//{
// //ArrayList tmp = _sparsedBuckets.Clone() as ArrayList;
// IEnumerator e = _sparsedBuckets.GetEnumerator();
// lock (_sparsedBuckets.SyncRoot)
// {
// while (e.MoveNext())
// {
// BucketsPack bPack = (BucketsPack)e.Current;
// ArrayList bucketIds = bPack.BucketIds.Clone() as ArrayList;
// foreach (int bucketId in bucketIds)
// {
// HashMapBucket current = new HashMapBucket(null, bucketId);
// if (!buckets.Contains(current))
// {
// ((BucketsPack)e.Current).BucketIds.Remove(bucketId);
// }
// else
// {
// HashMapBucket bucket = buckets[buckets.IndexOf(current)] as HashMapBucket;
// if (!bPack.Owner.Equals(new NodeIdentity(bucket.CurrentShard, GetShardPrimary(bucket.CurrentShard))))
// {
// //either i have become owner of the bucket or
// //some one else for e.g a replica node
// if (logger != null && logger.IsInfoEnabled)
// logger.Info(loggingModule + ".UpdateStateTxfer", bucket.BucketId + "bucket owner changed old :" + bPack.Owner + " new :" + bucket.CurrentShard);
// bPack.BucketIds.Remove(bucketId);
// }
// }
// }
// if (bPack.BucketIds.Count == 0)
// {
// //This owner has left.
// leavingShards.Add(bPack.Owner);
// }
// }
// foreach (NodeIdentity leavingShard in leavingShards)
// {
// BucketsPack bPack = new BucketsPack(null, leavingShard);
// _sparsedBuckets.Remove(bPack);
// }
// leavingShards.Clear();
// }
//}
if (_filledBuckets != null && _filledBuckets.Count > 0)
{
//ArrayList tmp = _filledBuckets.Clone() as ArrayList;
IEnumerator e = _filledBuckets.GetEnumerator();
lock (_filledBuckets.SyncRoot)
{
while (e.MoveNext())
{
BucketsPack bPack = (BucketsPack)e.Current;
ArrayList bucketIds = bPack.BucketIds.Clone() as ArrayList;
foreach (int bucketId in bucketIds)
{
HashMapBucket current = new HashMapBucket(null, bucketId);
if (!buckets.Contains(current))
{
((BucketsPack)e.Current).BucketIds.Remove(bucketId);
}
else
{
HashMapBucket bucket = buckets[buckets.IndexOf(current)] as HashMapBucket;
if (!bPack.Owner.Equals(new NodeIdentity(bucket.CurrentShard /*, GetShardPrimary(bucket.CurrentShard)*/)))
{
//either i have become owner of the bucket or
//some one else for e.g a replica node
bPack.BucketIds.Remove(bucketId);
}
}
}
if (bPack.BucketIds.Count == 0)
{
//This owner has left.
leavingShards.Add(bPack.Owner);
}
}
foreach (NodeIdentity leavingShard in leavingShards)
{
BucketsPack bPack = new BucketsPack(null, leavingShard);
_filledBuckets.Remove(bPack);
}
leavingShards.Clear();
}
}
//Now we add those buckets which we have to be state transferred
//and are not currently in our list
IEnumerator ie = buckets.GetEnumerator();
ArrayList loggableBuckets = new ArrayList();
while (ie.MoveNext())
{
HashMapBucket bucket = ie.Current as HashMapBucket;
if (Context.LocalShardName.Equals(bucket.FinalShard, StringComparison.OrdinalIgnoreCase) && Context.LocalShardName.Equals(bucket.CurrentShard, StringComparison.OrdinalIgnoreCase))
{
BucketsPack bPack = new BucketsPack(null, new NodeIdentity(bucket.CurrentShard /*, GetShardPrimary(bucket.CurrentShard)*/));
//if (IsSparsedBucket(bucket.BucketId, bPack.Owner))
//{
// int index = _sparsedBuckets.IndexOf(bPack);
// if (index != -1)
// {
// bPack = _sparsedBuckets[index] as BucketsPack;
// }
// else
// _sparsedBuckets.Add(bPack);
// if (!bPack.BucketIds.Contains(bucket.BucketId))
// {
// bPack.BucketIds.Add(bucket.BucketId);
// }
//}
//else
{
int index = _filledBuckets.IndexOf(bPack);
if (index != -1)
{
bPack = _filledBuckets[index] as BucketsPack;
}
else
{
_filledBuckets.Add(bPack);
}
if (!bPack.BucketIds.Contains(bucket.BucketId))
{
bPack.BucketIds.Add(bucket.BucketId);
loggableBuckets.Add(bucket.BucketId);
}
}
}
}
_startedFrom = StartLoggingOnReplica(loggableBuckets);
}
}
catch (NullReferenceException ex)
{
if (LoggerManager.Instance.StateXferLogger != null && LoggerManager.Instance.StateXferLogger.IsErrorEnabled)
{
LoggerManager.Instance.StateXferLogger.Error(loggingModule + ".UpdateStateTxfr", ex.ToString());
}
}
catch (Exception e)
{
if (LoggerManager.Instance.StateXferLogger != null && LoggerManager.Instance.StateXferLogger.IsErrorEnabled)
{
LoggerManager.Instance.StateXferLogger.Error(loggingModule + ".UpdateStateTxfr", e.ToString());
}
}
finally
{
if (LoggerManager.Instance.StateXferLogger != null && LoggerManager.Instance.StateXferLogger.IsInfoEnabled)
{
LoggerManager.Instance.StateXferLogger.Info(loggingModule + ".UpdateStateTxfr", " Pulsing waiting thread");
}
System.Threading.Monitor.PulseAll(_stateTxfrMutex);
}
}
return(true);
}
/// <summary>
/// Releases a bucket by setting its status again to functional. Only
/// node who has set its status to state trxfr can change its status.
/// </summary>
/// <param name="buckets"></param>
/// <param name="node"></param>
public void ReleaseBuckets(ArrayList buckets, Address requestingNode)
{
try
{
Sync.AcquireWriterLock(Timeout.Infinite);
try
{
if (buckets != null)
{
IEnumerator ie = buckets.GetEnumerator();
while (ie.MoveNext())
{
lock (InstalledHashMap.SyncRoot)
{
HashMapBucket bucket = (HashMapBucket)InstalledHashMap[(int)ie.Current];
if (requestingNode.Equals(bucket.TempAddress))
{
bucket.Status = BucketStatus.Functional;
//Change permnant address only when node who locked the bucket
//has sent request to release after he has transfered the bucket completely.
bucket.PermanentAddress = bucket.TempAddress;
}
//NCacheLog.Error("DistributionMgr.ReleaseBuckets", " Released bucket from InstalledHashMap " + bucket.ToString() + ". Requested by " + requestingNode);
}
}
}
if (_bucketsOwnershipMap != null)
{
ArrayList nodeBuckets = _bucketsOwnershipMap[requestingNode] as ArrayList;
if (nodeBuckets != null)
{
foreach (int bucketId in buckets)
{
int indexOfBucket = -1;
int startIndex = 0;
do
{
//TempFix: Updates status for multipile occurances of the same bucket in ownership map for replica.
indexOfBucket = nodeBuckets.IndexOf(new HashMapBucket(null, bucketId), startIndex);
if (indexOfBucket != -1)
{
HashMapBucket bucket = nodeBuckets[indexOfBucket] as HashMapBucket;
if (requestingNode.Equals(bucket.TempAddress))
{
bucket.Status = BucketStatus.Functional;
//Change permnant address only when node who locked the bucket
//has sent request to release after he has transfered the bucket completely.
bucket.PermanentAddress = requestingNode;
}
startIndex = indexOfBucket + 1;
}
} while (indexOfBucket >= 0);
}
}
}
}
finally
{
Sync.ReleaseWriterLock();
NotifyBucketUpdate();
}
}
catch (NullReferenceException e)
{
//Null is expected at start of cache
}
catch (Exception e)
{
NCacheLog.Error("DistributionMgr.ReleaseBuckets", e.ToString());
}
}
public override Hashtable GetBucketsOwnershipMap(ArrayList hashMap)
{
Hashtable bucketsOwnerShipMap = new Hashtable();
try
{
Sync.AcquireReaderLock(Timeout.Infinite);
Hashtable coordinatorNodesOwnershipMap = base.GetBucketsOwnershipMap(hashMap);
Hashtable currentOwnershipMap = _bucketsOwnershipMap != null?_bucketsOwnershipMap.Clone() as Hashtable : null;
ArrayList replicas = null;
Address partitionCoordinator = null;
if (coordinatorNodesOwnershipMap != null)
{
IDictionaryEnumerator ide = coordinatorNodesOwnershipMap.GetEnumerator();
while (ide.MoveNext())
{
partitionCoordinator = ide.Key as Address;
ArrayList coordinatorBuckets = ide.Value as ArrayList;
string subgroup = _subGroupMap[partitionCoordinator] as string;
if (subgroup != null)
{
replicas = _partitionNodesInfo[subgroup] as ArrayList;
if (replicas != null)
{
foreach (PartNodeInfo node in replicas)
{
if (node.IsCoordinator)
{
bucketsOwnerShipMap.Add(node.NodeAddress, coordinatorBuckets);
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("PoRDistMgr.GetBucketsOwnerShipMap", subgroup + ": " + node.NodeAddress.ToString() + " has got " + coordinatorBuckets.Count + " buckets");
}
}
else
{
ArrayList currentBuckets = currentOwnershipMap != null ? currentOwnershipMap[node.NodeAddress] as ArrayList : null;
ArrayList updatedBucketsList = new ArrayList();
if (currentBuckets != null)
{
//Node was already in the partitioned.
foreach (HashMapBucket bucket in currentBuckets)
{
//if bucket is not transferred to the replica yet then we
//change the temp address to make sure that if the coordinator
//of the partitioned is changed, it is reflected in the map.
if (coordinatorBuckets.Contains(bucket))
{
if (bucket.TempAddress != null && !bucket.PermanentAddress.Equals(bucket.TempAddress))
{
bucket.PermanentAddress = partitionCoordinator;
}
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("PoRDistMgr.GetBucketsOwnerShipMap", bucket.ToString() + " after coordinator left");
}
updatedBucketsList.Add(bucket.Clone());
}
}
//during loadbalancing; some new buckets may be assigned to a replica coordinator.
foreach (HashMapBucket coodinatorBucket in coordinatorBuckets)
{
if (!currentBuckets.Contains(coodinatorBucket))
{
HashMapBucket newNodeBucket = coodinatorBucket.Clone() as HashMapBucket;
newNodeBucket.PermanentAddress = partitionCoordinator;
newNodeBucket.TempAddress = node.NodeAddress;
//replica node need to state transfer from his coordinator.
newNodeBucket.Status = BucketStatus.NeedTransfer;
updatedBucketsList.Add(newNodeBucket);
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("PoRDistMgr.GetBucketsOwnerShipMap", newNodeBucket.ToString() + " new bucket assigned to replica");
}
}
}
}
else
{
//it is a new replica node,so we create a new bucket list for this node
foreach (HashMapBucket coodinatorBucket in coordinatorBuckets)
{
HashMapBucket newNodeBucket = coodinatorBucket.Clone() as HashMapBucket;
newNodeBucket.PermanentAddress = partitionCoordinator;
newNodeBucket.TempAddress = node.NodeAddress;
//replica node need to state transfer from his coordinator.
newNodeBucket.Status = BucketStatus.NeedTransfer;
updatedBucketsList.Add(newNodeBucket);
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("PoRDistMgr.GetBucketsOwnerShipMap", newNodeBucket.ToString() + " fresh replica");
}
}
}
if (NCacheLog.IsInfoEnabled)
{
NCacheLog.Info("PoRDistMgr.GetBucketsOwnerShipMap", subgroup + ": " + node.NodeAddress.ToString() + " has got " + updatedBucketsList.Count + " buckets");
}
bucketsOwnerShipMap.Add(node.NodeAddress, updatedBucketsList);
}
}
}
}
}
}
}
catch (Exception e)
{
NCacheLog.Error("PoRDistMgr.GetBucketsOwnerShipMap", e.ToString());
}
finally
{
Sync.ReleaseReaderLock();
}
return(bucketsOwnerShipMap);
}
public virtual HashMapBucket GetBucketForWait(int bucket, Address owner)
{
HashMapBucket bkt = _installedHashMap[bucket] as HashMapBucket;
return(bkt);
}
/// <summary>
/// A new map is required when a member leaves or joins the cluster.
/// This method returns a new map based on the input paramameters.
/// </summary>
/// <param name="member">Address of the member that has either left
/// or joined the cluster</param>
/// <param name="isNew">A flag. True if the member has joined otherwise false.</param>
/// <returns>A new hashmap instance</returns>
public virtual DistributionMaps GetMaps(DistributionInfoData distInfoData)
{
ArrayList tmpMap = null;
Hashtable bucketsOwnershipMap = null;
ArrayList partitionNodes = new ArrayList();
_sync.AcquireWriterLock(Timeout.Infinite);
try
{
if (_installedHashMap == null)
{
tmpMap = new ArrayList(TotalBuckets);
for (int i = 0; i < TotalBuckets; i++)
{
HashMapBucket bucket = new HashMapBucket(distInfoData.AffectedNode.NodeAddress, i, BucketStatus.Functional);
tmpMap.Add(bucket);
}
_existingMembers.Add(distInfoData.AffectedNode.NodeAddress);
_lastCreatedHashMap = tmpMap.Clone() as ArrayList;
bucketsOwnershipMap = GetBucketsOwnershipMap(_lastCreatedHashMap);
return(new DistributionMaps(_lastCreatedHashMap, bucketsOwnershipMap));
}
else if (_lastCreatedHashMap == null)
{
_lastCreatedHashMap = _installedHashMap.Clone() as ArrayList;
}
switch (distInfoData.ClustActivity)
{
case ClusterActivity.NodeJoin:
try
{
return(GetMapsOnNodeJoining(distInfoData));
}
catch (Exception e)
{
if (NCacheLog.IsErrorEnabled)
{
NCacheLog.Error("DistributionMgr.GetMaps()", e.ToString());
}
break;
}
case ClusterActivity.NodeLeave:
try
{
return(GetMapsOnNodeLeaving(distInfoData));
}
catch (Exception e)
{
if (NCacheLog.IsErrorEnabled)
{
NCacheLog.Error("DistributionMgr.GetMaps()", e.ToString());
}
break;
}
case ClusterActivity.None:
BalanceNodeMgr bnMgr = new BalanceNodeMgr(null);
DistributionMaps result = bnMgr.BalanceNodes(distInfoData, _lastCreatedHashMap, _bucketsStats, _existingMembers);
if (result.Hashmap != null)
{
_lastCreatedHashMap = result.Hashmap.Clone() as ArrayList;
result.BucketsOwnershipMap = GetBucketsOwnershipMap(_lastCreatedHashMap);
}
return(result);
default:
break;
}
}
finally
{
_sync.ReleaseWriterLock();
}
return(null);
}
