/// <summary>
/// Checks if any of the neighbors expired.
/// If so, it starts the process of their removal.
/// </summary>
public void CheckExpiredNeighbors()
{
log.Trace("()");
// If a neighbor server's LastRefreshTime is lower than this limit, it is expired.
DateTime limitLastRefreshTime = DateTime.UtcNow.AddSeconds(-Base.Configuration.NeighborProfilesExpirationTimeSeconds);
using (UnitOfWork unitOfWork = new UnitOfWork())
{
bool success = false;
DatabaseLock[] lockObjects = new DatabaseLock[] { UnitOfWork.NeighborLock, UnitOfWork.NeighborhoodActionLock };
using (IDbContextTransaction transaction = unitOfWork.BeginTransactionWithLock(lockObjects))
{
try
{
List <Neighbor> expiredNeighbors = unitOfWork.NeighborRepository.Get(n => n.LastRefreshTime < limitLastRefreshTime, null, true).ToList();
if (expiredNeighbors.Count > 0)
{
log.Debug("There are {0} expired neighbors.", expiredNeighbors.Count);
foreach (Neighbor neighbor in expiredNeighbors)
{
// This action will cause our profile server to erase all profiles of the neighbor that has been removed.
NeighborhoodAction action = new NeighborhoodAction()
{
ServerId = neighbor.NeighborId,
Timestamp = DateTime.UtcNow,
Type = NeighborhoodActionType.RemoveNeighbor,
TargetIdentityId = null,
AdditionalData = null
};
unitOfWork.NeighborhoodActionRepository.Insert(action);
}
unitOfWork.SaveThrow();
transaction.Commit();
}
else
{
log.Debug("No expired neighbors found.");
}
success = true;
}
catch (Exception e)
{
log.Error("Exception occurred: {0}", e.ToString());
}
if (!success)
{
log.Warn("Rolling back transaction.");
unitOfWork.SafeTransactionRollback(transaction);
}
unitOfWork.ReleaseLock(lockObjects);
}
}
log.Trace("(-)");
}
/// <summary>
/// Checks if any of the follower servers need refresh.
/// If so, a neighborhood action is created.
/// </summary>
public async Task CheckFollowersRefreshAsync()
{
log.Trace("()");
// If a follower server's LastRefreshTime is lower than this limit, it should be refreshed.
DateTime limitLastRefreshTime = DateTime.UtcNow.AddSeconds(-Config.Configuration.FollowerRefreshTimeSeconds);
using (UnitOfWork unitOfWork = new UnitOfWork())
{
DatabaseLock[] lockObjects = new DatabaseLock[] { UnitOfWork.FollowerLock, UnitOfWork.NeighborhoodActionLock };
await unitOfWork.AcquireLockAsync(lockObjects);
try
{
List <Follower> followersToRefresh = (await unitOfWork.FollowerRepository.GetAsync(f => f.LastRefreshTime < limitLastRefreshTime, null, true)).ToList();
if (followersToRefresh.Count > 0)
{
log.Debug("There are {0} followers that need refresh.", followersToRefresh.Count);
foreach (Follower follower in followersToRefresh)
{
NeighborhoodAction action = new NeighborhoodAction()
{
ServerId = follower.FollowerId,
Type = NeighborhoodActionType.RefreshProfiles,
Timestamp = DateTime.UtcNow,
ExecuteAfter = DateTime.UtcNow,
TargetIdentityId = null,
AdditionalData = null
};
await unitOfWork.NeighborhoodActionRepository.InsertAsync(action);
log.Debug("Refresh neighborhood action for follower ID '{0}' will be inserted to the database.", follower.FollowerId.ToHex());
}
await unitOfWork.SaveThrowAsync();
log.Debug("{0} new neighborhood actions saved to the database.", followersToRefresh.Count);
}
else
{
log.Debug("No followers need refresh now.");
}
}
catch (Exception e)
{
log.Error("Exception occurred: {0}", e.ToString());
}
unitOfWork.ReleaseLock(lockObjects);
}
log.Trace("(-)");
}
/// <summary>
/// Processes NeighbourhoodChangedNotificationRequest message from LOC server.
/// <para>Adds, changes, or deletes neighbor and possibly adds new neighborhood action to the database.</para>
/// </summary>
/// <param name="Client">TCP client who received the message.</param>
/// <param name="RequestMessage">Full request message.</param>
/// <returns>Response message to be sent to the client.</returns>
public async Task <LocProtocolMessage> ProcessMessageNeighbourhoodChangedNotificationRequestAsync(LocClient Client, LocProtocolMessage RequestMessage)
{
log.Trace("()");
LocProtocolMessage res = Client.MessageBuilder.CreateErrorInternalResponse(RequestMessage);
bool signalActionProcessor = false;
NeighbourhoodChangedNotificationRequest neighbourhoodChangedNotificationRequest = RequestMessage.Request.LocalService.NeighbourhoodChanged;
using (UnitOfWork unitOfWork = new UnitOfWork())
{
DatabaseLock[] lockObjects = new DatabaseLock[] { UnitOfWork.NeighborLock, UnitOfWork.NeighborhoodActionLock };
using (IDbContextTransaction transaction = await unitOfWork.BeginTransactionWithLockAsync(lockObjects))
{
bool success = false;
bool saveDb = false;
try
{
int neighborhoodSize = await unitOfWork.NeighborRepository.CountAsync();
foreach (NeighbourhoodChange change in neighbourhoodChangedNotificationRequest.Changes)
{
// We do ignore errors here for each individual change and just continue processing the next item from the list.
log.Trace("Neighborhood change type is {0}.", change.ChangeTypeCase);
switch (change.ChangeTypeCase)
{
case NeighbourhoodChange.ChangeTypeOneofCase.AddedNodeInfo:
case NeighbourhoodChange.ChangeTypeOneofCase.UpdatedNodeInfo:
{
bool isAdd = change.ChangeTypeCase == NeighbourhoodChange.ChangeTypeOneofCase.AddedNodeInfo;
NodeInfo nodeInfo = isAdd ? change.AddedNodeInfo : change.UpdatedNodeInfo;
// Check whether a proximity server is running on this node.
// If not, it is not interesting for us at all, skip it.
int proximityServerPort;
byte[] proximityServerId;
if (!HasProximityServerService(nodeInfo, out proximityServerPort, out proximityServerId))
{
break;
}
NodeContact contact = nodeInfo.Contact;
IPAddress ipAddress = new IPAddress(contact.IpAddress.ToByteArray());
Iop.Locnet.GpsLocation location = nodeInfo.Location;
int latitude = location.Latitude;
int longitude = location.Longitude;
AddOrChangeNeighborResult addChangeRes = await AddOrChangeNeighborAsync(unitOfWork, proximityServerId, ipAddress, proximityServerPort, latitude, longitude, neighborhoodSize);
neighborhoodSize = addChangeRes.NeighborhoodSize;
if (addChangeRes.SaveDb)
{
saveDb = true;
}
if (addChangeRes.SignalActionProcessor)
{
signalActionProcessor = true;
}
break;
}
case NeighbourhoodChange.ChangeTypeOneofCase.RemovedNodeId:
{
byte[] serverId = change.RemovedNodeId.ToByteArray();
bool serverIdValid = serverId.Length == ProtocolHelper.NetworkIdentifierLength;
if (!serverIdValid)
{
log.Error("Received invalid neighbor server ID '{0}' from LOC server.", serverId.ToHex());
break;
}
// Data processing.
Neighbor existingNeighbor = (await unitOfWork.NeighborRepository.GetAsync(n => n.NetworkId == serverId)).FirstOrDefault();
if (existingNeighbor != null)
{
log.Trace("Creating neighborhood action to deleting neighbor ID '{0}' from the database.", serverId.ToHex());
string neighborInfo = JsonConvert.SerializeObject(existingNeighbor);
// Delete neighbor completely.
// This will cause our proximity server to erase all activities of the neighbor that has been removed.
bool deleted = await unitOfWork.NeighborRepository.DeleteNeighborAsync(serverId, -1, true);
if (deleted)
{
// Add action that will contact the neighbor and ask it to stop sending updates.
// Note that the neighbor information will be deleted by the time this action
// is executed and this is why we have to fill in AdditionalData.
NeighborhoodAction stopUpdatesAction = new NeighborhoodAction()
{
ServerId = serverId,
Timestamp = DateTime.UtcNow,
Type = NeighborhoodActionType.StopNeighborhoodUpdates,
TargetActivityId = 0,
TargetActivityOwnerId = null,
ExecuteAfter = DateTime.UtcNow,
AdditionalData = neighborInfo
};
await unitOfWork.NeighborhoodActionRepository.InsertAsync(stopUpdatesAction);
signalActionProcessor = true;
saveDb = true;
}
else
{
log.Error("Failed to remove neighbor ID '{0}' from the database.", serverId.ToHex());
// This is actually bad, we failed to remove a record from the database, which should never happen.
// We try to insert action to remove this neighbor later, but adding the action might fail as well.
NeighborhoodAction action = new NeighborhoodAction()
{
ServerId = serverId,
Timestamp = DateTime.UtcNow,
Type = NeighborhoodActionType.RemoveNeighbor,
TargetActivityId = 0,
TargetActivityOwnerId = null,
AdditionalData = null
};
await unitOfWork.NeighborhoodActionRepository.InsertAsync(action);
signalActionProcessor = true;
saveDb = true;
}
}
else
{
log.Debug("Neighbor ID '{0}' not found, can not be removed.", serverId.ToHex());
// It can be the case that this node has not an associated proximity server, so in that case we should ignore it.
// If the node has an associated proximity server, then nothing bad really happens here if we have activities
// of such a neighbor in NeighborActivity table. Those entries will expire and will be deleted.
}
break;
}
default:
log.Error("Invalid neighborhood change type '{0}'.", change.ChangeTypeCase);
break;
}
}
if (saveDb)
{
await unitOfWork.SaveThrowAsync();
transaction.Commit();
}
success = true;
res = Client.MessageBuilder.CreateNeighbourhoodChangedNotificationResponse(RequestMessage);
}
catch (Exception e)
{
log.Error("Exception occurred: {0}", e.ToString());
}
if (!success)
{
log.Warn("Rolling back transaction.");
unitOfWork.SafeTransactionRollback(transaction);
}
unitOfWork.ReleaseLock(lockObjects);
}
}
if (signalActionProcessor)
{
NeighborhoodActionProcessor neighborhoodActionProcessor = (NeighborhoodActionProcessor)Base.ComponentDictionary[NeighborhoodActionProcessor.ComponentName];
neighborhoodActionProcessor.Signal();
}
log.Trace("(-):*.Response.Status={0}", res.Response.Status);
return(res);
}
/// <summary>
/// Processes update received from LOC server that informs the proximity server about a new neighbor server or a change in existing neighbor server contact information.
/// </summary>
/// <param name="UnitOfWork">Unit of work instance.</param>
/// <param name="ServerId">Network identifier of the neighbor server.</param>
/// <param name="IpAddress">IP address of the neighbor server.</param>
/// <param name="Port">Primary interface port of the neighbor server.</param>
/// <param name="Latitude">GPS location latitude of the neighbor server.</param>
/// <param name="Longitude">GPS location longitude of the neighbor server.</param>
/// <param name="NeighborhoodSize">Size of the proximity server's neighborhood at the moment the function is called.</param>
/// <returns>Information related to how should the caller proceed further, described in AddOrChangeNeighborResult structure.</returns>
/// <remarks>The caller is responsible for calling this function within a database transaction with NeighborLock and NeighborhoodActionLock locks.</remarks>
public async Task <AddOrChangeNeighborResult> AddOrChangeNeighborAsync(UnitOfWork UnitOfWork, byte[] ServerId, IPAddress IpAddress, int Port, int Latitude, int Longitude, int NeighborhoodSize)
{
log.Trace("(ServerId:'{0}',IpAddress:{1},Port:{2},Latitude:{3},Longitude:{4},NeighborhoodSize:{5})", ServerId.ToHex(), IpAddress, Port, Latitude, Longitude, NeighborhoodSize);
AddOrChangeNeighborResult res = new AddOrChangeNeighborResult();
res.NeighborhoodSize = NeighborhoodSize;
// Data validation.
bool serverIdValid = ServerId.Length == ProtocolHelper.NetworkIdentifierLength;
if (!serverIdValid)
{
log.Error("Received invalid neighbor server ID '{0}' from LOC server.", ServerId.ToHex());
res.Error = true;
log.Trace("(-):*.Error={0},*.SaveDb={1},*.SignalActionProcessor={2},*.NeighborhoodSize={3}", res.Error, res.SaveDb, res.SignalActionProcessor, res.NeighborhoodSize);
return(res);
}
bool portValid = (0 < Port) && (Port <= 65535);
if (!portValid)
{
log.Error("Received invalid neighbor server port '{0}' from LOC server.", Port);
res.Error = true;
log.Trace("(-):*.Error={0},*.SaveDb={1},*.SignalActionProcessor={2},*.NeighborhoodSize={3}", res.Error, res.SaveDb, res.SignalActionProcessor, res.NeighborhoodSize);
return(res);
}
IopProtocol.GpsLocation location = new IopProtocol.GpsLocation(Latitude, Longitude);
if (!location.IsValid())
{
log.Error("Received invalid neighbor server location '{0}' from LOC server.", location);
res.Error = true;
log.Trace("(-):*.Error={0},*.SaveDb={1},*.SignalActionProcessor={2},*.NeighborhoodSize={3}", res.Error, res.SaveDb, res.SignalActionProcessor, res.NeighborhoodSize);
return(res);
}
// Data processing.
Neighbor existingNeighbor = (await UnitOfWork.NeighborRepository.GetAsync(n => n.NetworkId == ServerId)).FirstOrDefault();
if (existingNeighbor == null)
{
// New neighbor server.
if (NeighborhoodSize < Config.Configuration.MaxNeighborhoodSize)
{
// We have not reached the maximal size of the neighborhood yet, the server can be added.
log.Trace("New neighbor ID '{0}' detected, IP address {1}, port {2}, latitude {3}, longitude {4}.", ServerId.ToHex(), IpAddress, Port, Latitude, Longitude);
// Add neighbor to the database of neighbors.
// The neighbor is not initialized, so we will not allow it to send us
// any updates. First, we need to contact it and start the neighborhood initialization process.
Neighbor neighbor = new Neighbor()
{
NetworkId = ServerId,
IpAddress = IpAddress.GetAddressBytes(),
PrimaryPort = Port,
NeighborPort = null,
LocationLatitude = location.Latitude,
LocationLongitude = location.Longitude,
LastRefreshTime = DateTime.UtcNow,
Initialized = false,
SharedActivities = 0
};
await UnitOfWork.NeighborRepository.InsertAsync(neighbor);
res.NeighborhoodSize++;
// This action will cause our proximity server to contact the new neighbor server and ask it to share its activity database,
// i.e. the neighborhood initialization process will be started.
// We set a delay depending on the number of neighbors, so that a new server joining a neighborhood is not overwhelmed with requests.
int delay = RandomSource.Generator.Next(0, 3 * res.NeighborhoodSize);
NeighborhoodAction action = new NeighborhoodAction()
{
ServerId = ServerId,
Timestamp = DateTime.UtcNow,
Type = NeighborhoodActionType.AddNeighbor,
ExecuteAfter = DateTime.UtcNow.AddSeconds(delay),
TargetActivityId = 0,
TargetActivityOwnerId = null,
AdditionalData = null,
};
await UnitOfWork.NeighborhoodActionRepository.InsertAsync(action);
res.SignalActionProcessor = true;
res.SaveDb = true;
}
else
{
log.Error("Unable to add new neighbor ID '{0}', the proximity server reached its neighborhood size limit {1}.", ServerId.ToHex(), Config.Configuration.MaxNeighborhoodSize);
}
}
else
{
// This is a neighbor we already know about. Just check that its information is up to date and if not, update it.
IPAddress existingNeighborIpAddress = new IPAddress(existingNeighbor.IpAddress);
if (!existingNeighborIpAddress.Equals(IpAddress))
{
log.Trace("Existing neighbor ID '{0}' changed its IP address from {1} to {2}.", ServerId.ToHex(), existingNeighborIpAddress, IpAddress);
existingNeighbor.IpAddress = IpAddress.GetAddressBytes();
}
if (existingNeighbor.PrimaryPort != Port)
{
// Primary port was change, so we also expect that the neighbors interface port was changed as well.
log.Trace("Existing neighbor ID '{0}' changed its primary port from {1} to {2}, invalidating neighbors interface port as well.", ServerId.ToHex(), existingNeighbor.PrimaryPort, Port);
existingNeighbor.PrimaryPort = Port;
existingNeighbor.NeighborPort = null;
}
if (existingNeighbor.LocationLatitude != location.Latitude)
{
log.Trace("Existing neighbor ID '{0}' changed its latitude from {1} to {2}.", ServerId.ToHex(), existingNeighbor.LocationLatitude, location.Latitude);
existingNeighbor.LocationLatitude = Latitude;
}
if (existingNeighbor.LocationLongitude != location.Longitude)
{
log.Trace("Existing neighbor ID '{0}' changed its longitude from {1} to {2}.", ServerId.ToHex(), existingNeighbor.LocationLongitude, location.Longitude);
existingNeighbor.LocationLongitude = Longitude;
}
// We consider a fresh LOC info to be accurate, so we do not want to delete the neighbors received here
// and hence we update their refresh time.
existingNeighbor.LastRefreshTime = DateTime.UtcNow;
UnitOfWork.NeighborRepository.Update(existingNeighbor);
res.SaveDb = true;
}
log.Trace("(-):*.Error={0},*.SaveDb={1},*.SignalActionProcessor={2},*.NeighborhoodSize={3}", res.Error, res.SaveDb, res.SignalActionProcessor, res.NeighborhoodSize);
return(res);
}
/// <summary>
/// Checks if any of the neighbors expired.
/// If so, it starts the process of their removal.
/// </summary>
public async Task CheckExpiredNeighborsAsync()
{
log.Trace("()");
// If a neighbor server's LastRefreshTime is lower than this limit, it is expired.
DateTime limitLastRefreshTime = DateTime.UtcNow.AddSeconds(-Config.Configuration.NeighborExpirationTimeSeconds);
List <byte[]> neighborsToDeleteIds = new List <byte[]>();
using (UnitOfWork unitOfWork = new UnitOfWork())
{
bool success = false;
DatabaseLock[] lockObjects = new DatabaseLock[] { UnitOfWork.NeighborLock, UnitOfWork.NeighborhoodActionLock };
using (IDbContextTransaction transaction = await unitOfWork.BeginTransactionWithLockAsync(lockObjects))
{
try
{
List <Neighbor> expiredNeighbors = (await unitOfWork.NeighborRepository.GetAsync(n => n.LastRefreshTime < limitLastRefreshTime, null, true)).ToList();
if (expiredNeighbors.Count > 0)
{
bool saveDb = false;
log.Debug("There are {0} expired neighbors.", expiredNeighbors.Count);
foreach (Neighbor neighbor in expiredNeighbors)
{
int unprocessedRemoveNeighborActions = await unitOfWork.NeighborhoodActionRepository.CountAsync(a => (a.ServerId == neighbor.NetworkId) && (a.Type == NeighborhoodActionType.RemoveNeighbor));
if (unprocessedRemoveNeighborActions == 0)
{
// This action will cause our proximity server to erase all profiles of the neighbor that has been removed.
NeighborhoodAction action = new NeighborhoodAction()
{
ServerId = neighbor.NetworkId,
Timestamp = DateTime.UtcNow,
Type = NeighborhoodActionType.RemoveNeighbor,
TargetActivityId = 0,
TargetActivityOwnerId = null,
AdditionalData = null
};
await unitOfWork.NeighborhoodActionRepository.InsertAsync(action);
saveDb = true;
}
else
{
log.Debug("There is an unprocessed RemoveNeighbor neighborhood action for neighbor ID '{0}'. Neighbor will be deleted.", neighbor.NetworkId.ToHex());
neighborsToDeleteIds.Add(neighbor.NetworkId);
}
}
if (saveDb)
{
await unitOfWork.SaveThrowAsync();
transaction.Commit();
}
}
else
{
log.Debug("No expired neighbors found.");
}
success = true;
}
catch (Exception e)
{
log.Error("Exception occurred: {0}", e.ToString());
}
if (!success)
{
log.Warn("Rolling back transaction.");
unitOfWork.SafeTransactionRollback(transaction);
}
unitOfWork.ReleaseLock(lockObjects);
}
if (neighborsToDeleteIds.Count > 0)
{
log.Debug("There are {0} neighbors to be deleted.", neighborsToDeleteIds.Count);
foreach (byte[] neighborToDeleteId in neighborsToDeleteIds)
{
if (await unitOfWork.NeighborRepository.DeleteNeighborAsync(neighborToDeleteId))
{
log.Debug("Neighbor ID '{0}' deleted.", neighborToDeleteId.ToHex());
}
else
{
log.Warn("Unable to delete neighbor ID '{0}'.", neighborToDeleteId.ToHex());
}
}
}
else
{
log.Debug("No neighbors to delete now.");
}
}
log.Trace("(-)");
}
/// <summary>
/// Checks if any of the follower servers need refresh. If so, a neighborhood action is created.
/// <para>This function also checks if there are unprocessed refresh neighborhood actions
/// and if there are 3 such requests already, the follower is deleted as it is considered as unresponsive for too long.</para>
/// </summary>
public async Task CheckFollowersRefreshAsync()
{
log.Trace("()");
// If a follower server's LastRefreshTime is lower than this limit, it should be refreshed.
DateTime limitLastRefreshTime = DateTime.UtcNow.AddSeconds(-Config.Configuration.FollowerRefreshTimeSeconds);
List <byte[]> followersToDeleteIds = new List <byte[]>();
using (UnitOfWork unitOfWork = new UnitOfWork())
{
DatabaseLock[] lockObjects = new DatabaseLock[] { UnitOfWork.FollowerLock, UnitOfWork.NeighborhoodActionLock };
await unitOfWork.AcquireLockAsync(lockObjects);
try
{
List <Follower> followersToRefresh = (await unitOfWork.FollowerRepository.GetAsync(f => f.LastRefreshTime < limitLastRefreshTime, null, true)).ToList();
if (followersToRefresh.Count > 0)
{
bool saveDb = false;
int actionsInserted = 0;
log.Debug("There are {0} followers that need refresh.", followersToRefresh.Count);
foreach (Follower follower in followersToRefresh)
{
int unprocessedRefreshProfileActions = await unitOfWork.NeighborhoodActionRepository.CountAsync(a => (a.ServerId == follower.NetworkId) && (a.Type == NeighborhoodActionType.RefreshNeighborStatus));
if (unprocessedRefreshProfileActions < 3)
{
NeighborhoodAction action = new NeighborhoodAction()
{
ServerId = follower.NetworkId,
Type = NeighborhoodActionType.RefreshNeighborStatus,
Timestamp = DateTime.UtcNow,
ExecuteAfter = DateTime.UtcNow,
TargetActivityId = 0,
TargetActivityOwnerId = null,
AdditionalData = null
};
await unitOfWork.NeighborhoodActionRepository.InsertAsync(action);
log.Debug("Refresh neighborhood action for follower ID '{0}' will be inserted to the database.", follower.NetworkId.ToHex());
saveDb = true;
actionsInserted++;
}
else
{
log.Debug("There are {0} unprocessed RefreshNeighborStatus neighborhood actions for follower ID '{1}'. Follower will be deleted.", unprocessedRefreshProfileActions, follower.NetworkId.ToHex());
followersToDeleteIds.Add(follower.NetworkId);
}
}
if (saveDb)
{
await unitOfWork.SaveThrowAsync();
log.Debug("{0} new neighborhood actions saved to the database.", actionsInserted);
}
}
else
{
log.Debug("No followers need refresh now.");
}
}
catch (Exception e)
{
log.Error("Exception occurred: {0}", e.ToString());
}
unitOfWork.ReleaseLock(lockObjects);
if (followersToDeleteIds.Count > 0)
{
log.Debug("There are {0} followers to be deleted.", followersToDeleteIds.Count);
foreach (byte[] followerToDeleteId in followersToDeleteIds)
{
Iop.Shared.Status status = unitOfWork.FollowerRepository.DeleteFollowerAsync(followerToDeleteId).Result;
if (status == Iop.Shared.Status.Ok)
{
log.Debug("Follower ID '{0}' deleted.", followerToDeleteId.ToHex());
}
else
{
log.Warn("Unable to delete follower ID '{0}', error code {1}.", followerToDeleteId.ToHex(), status);
}
}
}
else
{
log.Debug("No followers to delete now.");
}
}
log.Trace("(-)");
}