public void StatusUpdateTest()
{
// sample message (this caused deserialization issues)
var message = new StatusUpdateMessage()
{
InstanceHash = new Hash(new byte[] { 1, 2, 3 }),
KnownPackages = new PackageStatus[0],
KnownPeers = new DiscoveryPeerData[] {
new DiscoveryPeerData()
{
ServiceEndpoint = new IPEndPoint(IPAddress.Parse("192.168.0.110"), 1234),
PeerId = new Hash(new byte[] { 4, 5, 6 })
}
},
ServicePort = 5432,
PeerEndpoint = new IPEndPoint(IPAddress.Parse("192.168.0.109"), 5678)
};
// serialize/deserialize
IMessageSerializer serializer = new ProtoBufMessageSerializer();
byte[] bytes = serializer.Serialize((object)message, typeof(StatusUpdateMessage));
var des = serializer.Deserialize <StatusUpdateMessage>(bytes);
// compare
Assert.NotNull(des);
Assert.Equal(message.ServicePort, des.ServicePort);
Assert.Equal(message.InstanceHash, des.InstanceHash);
Assert.Equal(message.PeerEndpoint, des.PeerEndpoint);
Assert.NotNull(des.KnownPeers);
Assert.Equal(message.KnownPeers, des.KnownPeers, DiscoveryPeerData.Comparer);
}
private void SendStatusUpdateInternal()
{
logger.LogTrace("Sending cluster update to all peers.");
Task.Run(() =>
{
peerRegistry.ImmutablePeers.AsParallel()
.Where(p => !p.IsLoopback)
.ForAll(p =>
{
StatusUpdateMessage response;
try
{
StatusUpdateMessage statusMessage = CreateStatusUpdateMessage(p.ServiceEndPoint);
response = client.GetStatus(p.ServiceEndPoint, statusMessage);
}
catch (Exception e)
{
logger.LogWarning("Communication failed with peer {0} at {1:s}: {2}", p.ServiceEndPoint, p.PeerId, e.Message);
p.ClientHasFailed();
return;
}
logger.LogTrace("Getting status from peer {0:s} at {1}", p.PeerId, p.ServiceEndPoint);
ProcessDiscoveryMessage(response, p.ServiceEndPoint.Address, p.PeerId);
});
}).ContinueWith(t =>
{
if (t.IsFaulted)
{
logger.LogError(t.Exception, "Peer update status failed.");
}
});
}
public static StatusUpdateMessage Deserialize(byte[] bytes)
{
MemoryStream outMemStream = new MemoryStream(bytes, 0, bytes.Length);
StatusUpdateMessage updateMessage = (StatusUpdateMessage)Serializer.Deserialize <StatusUpdateMessage>(outMemStream);
outMemStream.Close();
return(updateMessage);
}
public static void GenerateLocal()
{
foreach (KeyValuePair <string, Tile> kV in tiles)
{
try
{
DestroyImmediate(kV.Value.gameObject);
}
catch (MissingReferenceException)
{
Debug.LogWarning("Trying to access dead object...");
}
}
tiles.Clear();
int jobCount = 0;
//Create new Tiles
for (int i = -tileRadius; i <= tileRadius; i++)
{
for (int j = -tileRadius; j <= tileRadius; j++)
{
StatusUpdateMessage msg = new StatusUpdateMessage(jobCount, i + "," + j);
TodoItem worker = msg.AddTodo(Job.Worker);
//worker.AddTodo(Job.CreateTile);
worker.AddTodo(Job.StartOSMQuery);
worker.AddTodo(Job.StartProcedural);
worker.AddTodo(Job.ProceduralPreparation);
worker.AddTodo(Job.CreateTerrain);
worker.AddTodo(Job.MeshPreparation);
worker.AddTodo(Job.TileQuad);
worker.AddTodo(Job.River);
worker.AddTodo(Job.Ways);
worker.AddTodo(Job.CreateBuildingMesh);
worker.AddTodo(Job.FillMeshDivideMaterials);
worker.AddTodo(Job.GarbageCollection);
worker.AddTodo(Job.ProceduralDone);
// Add StatusUpdateMessage to Dictionary
Tile newTile = Tile.CreateTileGO(i, j, LOD);
msg.Start();
newTile.SetJobInfo(jobCount, msg);
tiles.Add(i + ":" + j, newTile);
SimpleClient.jobStatus.Add(jobCount, msg);
jobCount++;
}
}
foreach (KeyValuePair <string, Tile> kV in tiles)
{
kV.Value.StartQuery();
}
}
private void SendStatusUpdateInternal()
{
int stamp = statusVersion;
TimeSpan time = clock.Time;
TimeSpan timeMaximum = time.Subtract(appInfo.NetworkSettings.PeerStatusUpdateStatusMaximumTimer);
TimeSpan timeFast = time.Subtract(appInfo.NetworkSettings.PeerStatusUpdateStatusFastTimer);
// get clients that should be updated
var allRemotePeers = peerRegistry
.ImmutablePeers
.Where(p => !p.IsLoopback)
.Where(p =>
// maximum time to update expired
p.Status.LastKnownStateUpdateAttemptTime > timeMaximum
// data has changed and minimum time to update has expired
|| (p.Status.LastKnownStateUpdateAttemptTime > timeFast && p.Status.LastKnownStateUdpateVersion < stamp)
);
// recap
int allRemotePeersCount = allRemotePeers.Count();
if (allRemotePeersCount == 0)
{
logger.LogTrace($"Sending cluster update skip - no peers.");
return;
}
logger.LogTrace($"Sending cluster update to all {allRemotePeersCount} peers.");
// run update
Task.Run(() =>
{
allRemotePeers.AsParallel()
.ForAll(p =>
{
StatusUpdateMessage response;
try
{
StatusUpdateMessage statusMessage = CreateStatusUpdateMessage(p.ServiceEndPoint);
response = client.GetStatus(p.ServiceEndPoint, statusMessage);
}
catch (Exception e)
{
logger.LogWarning("Communication failed with peer {0}: {1}", p.ServiceEndPoint, e.Message);
OnPeerStatusUpdateFail(p);
return;
}
logger.LogTrace("Got status update from {0}", p.ServiceEndPoint);
ProcessStatusUpdateMessage(response, p.ServiceEndPoint.Address);
});
}).ContinueWith(t =>
{
if (t.IsFaulted)
{
logger.LogError(t.Exception, "Peer update status failed.");
}
});
}
public SceneMessage(int Job_ID, string messageText, Scene scene, StatusUpdateMessage statusUpdateMessage, SerializationMethod method)
{
this.Job_ID = Job_ID;
this.method = method;
this.messageText = messageText;
this.scene = scene;
this.sceneBytes = SceneFileToByteArray(this.scene, method);
this.statusUpdateMessage = statusUpdateMessage;
}
public OSMJobMessage(int x, int y, double tileWidth, double originLongitude, double originLatitude, string replyQueueName, string statusUpdateQueue, StatusUpdateMessage statusUpdateMessage, SerializationMethod method)
{
this.x = x;
this.y = y;
this.tileWidth = tileWidth;
this.originLongitude = originLongitude;
this.originLatitude = originLatitude;
this.replyToQueue = replyQueueName;
this.statusUpdateQueue = statusUpdateQueue;
this.statusUpdateMessage = statusUpdateMessage;
this.method = method;
}
public StatusUpdateMessage StatusUpdate([FromBody] StatusUpdateMessage request)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
var address = RemoteIpAddress;
peersCluster.ProcessDiscoveryMessage(request, address, PeerId);
var response = peersCluster.CreateStatusUpdateMessage(new IPEndPoint(address, request.ServicePort));
return(response);
}
public StatusUpdateMessage CreateStatusUpdateMessage(IPEndPoint endpoint)
{
lock (clusterNodeLock)
{
var result = new StatusUpdateMessage
{
InstanceHash = appInfo.InstanceHash.Hash,
KnownPackages = packageRegistry.ImmutablePackagesStatuses,
KnownPeers = peerRegistry.ImmutablePeersDiscoveryData,
ServicePort = appInfo.NetworkSettings.TcpServicePort,
PeerEndpoint = endpoint
};
return(result);
}
}
public void StatusUpdate(StatusUpdateMessage message)
{
if (message.ProgressValue > App._PROGRESS_BAR_HIDE)
{
UpdateProgress(message.ProgressValue);
}
else
{
ProgressBarVisibility = Visibility.Hidden;
}
if (message.StatusMessage != null)
{
StatusMessage = message.StatusMessage;
}
ShowBusy(message.ShowBusy);
}
private void DrawTodoItem(StatusUpdateMessage item)
{
switch (item.status)
{
case Status.PENDING:
break;
case Status.IN_PROGRESS:
GUILayout.Button(item.name, GUILayout.Width((float)item.Duration() * zoomFactor));
break;
case Status.DONE:
GUILayout.Button(item.name, GUILayout.Width((float)item.Duration() * zoomFactor));
break;
default:
break;
}
}
public async Task <RestResponseStatus> PostDeviceStatusUpdate(RestSession session, DeviceStatusUpdate statusUpdate)
{
var headers = SessionToHeaders(session);
var message = new StatusUpdateMessage
{
Country = statusUpdate.Country.ToString(),
Traffic = statusUpdate.Traffic,
Device = new Device
{
ClassName = "Device",
Type = "Pointer",
ObjectId = session.ObjectId
}
};
var content = JsonSerializer.Serialize(message);
var result = await _restService.SendRestPostRequest(PostDeviceStatusLink, headers, content);
return(result.Status);
}
public void ProcessDiscoveryMessage(StatusUpdateMessage message, IPAddress address, Hash peerId)
{
// is this request from myself?
bool isLoopback = appInfo.InstanceHash.Hash.Equals(message.InstanceHash);
var endPoint = new IPEndPoint(address, message.ServicePort);
// register peers
IEnumerable <PeerInfo> discoveredPeers = (message.KnownPeers ?? new DiscoveryPeerData[0])
.Select(kp => new PeerInfo(kp.PeerId, kp.ServiceEndpoint, isOtherPeerDiscovery: true)) // peers known to peer we're communicating with
.Concat(new[] { new PeerInfo(peerId, endPoint, isDirectDiscovery: true, isLoopback: isLoopback) }); // direct peer we're communicating with
// if one of known peers is me, mark as loopback
discoveredPeers = discoveredPeers.Select(discoveredPeer =>
{
if (discoveredPeer.ServiceEndPoint.Equals(message.PeerEndpoint))
{
discoveredPeer.IsLoopback = true;
discoveredPeer.IsDirectDiscovery = true;
}
return(discoveredPeer);
});
peerRegistry.RegisterPeers(discoveredPeers);
if (!peerRegistry.TryGetPeer(peerId, out PeerInfo peer))
{
throw new InvalidOperationException($"Can't find peer in internal registry: {peerId} {address}");
}
// update known packages if different
peer.ReplaceKnownPackages(message.KnownPackages ?? Array.Empty <PackageStatus>());
// register discovered packages
if (message.KnownPackages?.Any() == true)
{
packageRegistry.RegisterDiscoveredPackages(message.KnownPackages.Select(kp => new DiscoveredPackage(endPoint, kp.Meta)));
}
// mark as success peer
peer.ClientHasSuccess();
}
public void ProcessStatusUpdateMessage(StatusUpdateMessage message, IPAddress address)
{
// is this request from myself?
bool isLoopback = appInfo.InstanceHash.Hash.Equals(message.InstanceHash);
var endPoint = new IPEndPoint(address, message.ServicePort);
// register peers
IEnumerable <PeerUpdateInfo> discoveredPeers = (message.KnownPeers ?? new DiscoveryPeerData[0])
.Select(kp => new PeerUpdateInfo(kp.ServiceEndpoint, PeerDiscoveryMode.OtherPeerDiscovery, clock.ConvertToLocal(message.Clock, kp.LastSuccessCommunication))) // peers known to peer we're communicating with
.Concat(new[] { new PeerUpdateInfo(endPoint, PeerDiscoveryMode.DirectDiscovery, clock.Time) }); // direct peer we're communicating with
peerRegistry.UpdatePeers(discoveredPeers);
if (!peerRegistry.TryGetPeer(endPoint, out PeerInfo peer))
{
throw new InvalidOperationException($"Can't find peer in internal registry: {endPoint}");
}
// don't process requests from myself
if (peer.IsLoopback)
{
return;
}
// update known packages if different
peer.ReplaceKnownPackages(message.KnownPackages ?? Array.Empty <PackageStatus>());
// register discovered packages
if (message.KnownPackages?.Any() == true)
{
packageRegistry.RegisterDiscoveredPackages(message.KnownPackages.Select(kp => new DiscoveredPackage(endPoint, kp.Meta)));
}
// mark peer have new information
OnPeerStatusUpdateSuccess(peer);
}
public static StatusUpdateMessage Merge(StatusUpdateMessage local, StatusUpdateMessage remote)
{
switch (local.status)
{
case Status.PENDING:
#region Pending
switch (remote.status)
{
case Status.PENDING:
// Nothing to be done.
break;
case Status.IN_PROGRESS:
local.status = Status.IN_PROGRESS;
local.startTime = remote.startTime;
break;
case Status.DONE:
local.status = Status.DONE;
local.startTime = remote.startTime;
local.endTime = remote.endTime;
break;
default:
break;
}
#endregion Pending
break;
case Status.IN_PROGRESS:
#region In Progress
switch (remote.status)
{
case Status.PENDING:
// Nothing to be done.
break;
case Status.IN_PROGRESS:
if (local.startTime != remote.startTime)
{
if (local.startTime.CompareTo(remote.startTime) > 0)
{
local.startTime = remote.startTime;
}
}
break;
case Status.DONE:
local.status = Status.DONE;
if (local.startTime != remote.startTime)
{
if (local.startTime.CompareTo(remote.startTime) > 0)
{
local.startTime = remote.startTime;
}
}
local.endTime = remote.endTime;
break;
default:
break;
}
#endregion In Progress
break;
case Status.DONE:
#region done
switch (remote.status)
{
case Status.PENDING:
// nothing to be done
break;
case Status.IN_PROGRESS:
// nothing to be done
break;
case Status.DONE:
if (local.startTime != remote.startTime)
{
if (local.startTime.CompareTo(remote.startTime) > 0)
{
local.startTime = remote.startTime;
}
}
if (local.endTime != remote.startTime)
{
if (local.startTime.CompareTo(remote.startTime) > 0)
{
local.startTime = remote.startTime;
}
}
break;
default:
break;
}
#endregion done
break;
default:
break;
}
foreach (string item in local.childTodos)
{
TodoItem newLocal = local.childDict[item];
TodoItem newRemote = remote.childDict[item];
local.childDict[item] = TodoItem.Merge(newLocal, newRemote);
}
return(local);
}
public void SetJobInfo(int jobCount, StatusUpdateMessage msg)
{
this.jobNumber = jobCount;
this.msg = msg;
}
private static void CheckDone()
{
if (stopTime == null)
{
bool allJobsDone = true;
foreach (StatusUpdateMessage item in SimpleClient.jobStatus.Values)
{
if (item.status != Status.DONE)
{
allJobsDone = false;
break;
}
}
#region Collect Statistics
if (allJobsDone && SimpleClient.jobStatus.Count > 0)
{
stopTime = TimeStamp.Now();
Debug.Log("All Jobs Done");
Debug.Log("Gathering result...");
Dictionary <string, double> result = new Dictionary <string, double>();
foreach (StatusUpdateMessage item in SimpleClient.jobStatus.Values)
{
foreach (TodoItem cildItem in item.childDict.Values)
{
foreach (TodoItem childTodo in cildItem.childDict.Values)
{
if (!result.ContainsKey(childTodo.name))
{
result.Add(childTodo.name, 0);
}
result[childTodo.name] += childTodo.Duration();
}
if (!result.ContainsKey(cildItem.name))
{
result.Add(cildItem.name, 0);
}
result[cildItem.name] += cildItem.Duration();
}
if (!result.ContainsKey(item.name))
{
result.Add(item.name, 0);
}
result[item.name] += item.Duration();
}
StringBuilder sb = new StringBuilder();
sb.Append("StartTime,").Append(startTime).AppendLine();
sb.Append("StopTime,").Append(stopTime).AppendLine();
sb.Append("OriginLon,").Append(TileManager.OriginLongitude).AppendLine();
sb.Append("OriginLat,").Append(TileManager.OriginLatitude).AppendLine();
sb.Append("TileRadius, ").Append(TileManager.tileRadius).AppendLine();
sb.Append("TileWidth,").Append(TileManager.TileWidth).AppendLine();
StatusUpdateMessage msg = SimpleClient.jobStatus[0];
foreach (TodoItem cildItem in msg.childDict.Values)
{
sb.Append(cildItem.name).Append(", ").Append(result[cildItem.name]).Append("\n");
foreach (TodoItem childTodo in cildItem.childDict.Values)
{
sb.Append(childTodo.name).Append(", ").Append(result[childTodo.name]).Append("\n");
}
}
Debug.Log(sb.ToString());
string logPath = Application.dataPath + "/Logs/";
if (!Directory.Exists(logPath))
{
Directory.CreateDirectory(logPath);
}
System.IO.File.WriteAllText(logPath + "Session.txt", sb.ToString());
}
#endregion // Collect Statistics
}
}
private void JobCreationGUI()
{
EditorGUILayout.BeginVertical("box");
TileManager.TileWidth = (double)EditorGUILayout.FloatField("TileWidth", (float)TileManager.TileWidth);
TileManager.tileRadius = EditorGUILayout.IntField("TileRadius", TileManager.tileRadius);
TileManager.LOD = EditorGUILayout.IntField("LOD", TileManager.LOD);
TileManager.OriginLatitude = (double)EditorGUILayout.FloatField("OriginLatitude", (float)TileManager.OriginLatitude);
TileManager.OriginLongitude = (double)EditorGUILayout.FloatField("OriginLongitude", (float)TileManager.OriginLongitude);
EditorGUILayout.EndVertical();
client.method = (SerializationMethod)EditorGUILayout.EnumPopup("SerializationMethod", client.method);
if (GUILayout.Button("Send OSM-Job-Messages"))
{
startTime = TimeStamp.Now();
stopTime = null;
done = false;
generateLocal = false;
client.SendOSMJobMessages(
jobQueueName,
replyQueueName,
statusUpdateQueueName,
TileManager.tileRadius,
TileManager.TileWidth,
TileManager.OriginLongitude,
TileManager.OriginLatitude,
client.method);
}
sequential = EditorGUILayout.ToggleLeft("Sequential", sequential);
if (GUILayout.Button("Generate Local"))
{
startTime = TimeStamp.Now();
stopTime = null;
done = false;
generateLocal = true;
if (sequential)
{
tiles = new List <Tile>();
jobs = 0;
for (int i = -TileManager.tileRadius; i <= TileManager.tileRadius; i++)
{
for (int j = -TileManager.tileRadius; j <= TileManager.tileRadius; j++)
{
StatusUpdateMessage msg = new StatusUpdateMessage(jobs, i + "," + j);
TodoItem worker = msg.AddTodo(Job.Worker);
//worker.AddTodo(Job.CreateTile);
worker.AddTodo(Job.StartOSMQuery);
worker.AddTodo(Job.StartProcedural);
worker.AddTodo(Job.ProceduralPreparation);
worker.AddTodo(Job.CreateTerrain);
worker.AddTodo(Job.MeshPreparation);
worker.AddTodo(Job.TileQuad);
worker.AddTodo(Job.River);
worker.AddTodo(Job.Ways);
worker.AddTodo(Job.CreateBuildingMesh);
worker.AddTodo(Job.FillMeshDivideMaterials);
worker.AddTodo(Job.GarbageCollection);
worker.AddTodo(Job.ProceduralDone);
SimpleClient.jobStatus.Add(jobs, msg);
Tile newTile = Tile.CreateTileGO(i, j, 5);
tiles.Add(newTile);
newTile.SetJobInfo(jobs, msg);
jobs++;
}
}
// Start first Tile-Generation
// Next Tile will be started after "GenerationDone" of first Tile
tiles[0].ProceduralDoneLocal += GenerationDone;
tiles[0].StartQuery();
}
else
{
// Start parallel Generation of Tiles locally
TileManager.GenerateLocal();
}
}
#region ScaleWorkers
int newNumberOfWorkers = EditorGUILayout.IntSlider("Number of Workers", numberOfWorkers, 1, 100);
if (newNumberOfWorkers != numberOfWorkers)
{
if (newNumberOfWorkers > numberOfWorkers)
{
Debug.Log("Scaling UP number of workers (from " + numberOfWorkers + " to " + newNumberOfWorkers + ")");
}
else
{
Debug.Log("Scaling DOWN number of workers (from " + numberOfWorkers + " to " + newNumberOfWorkers + ")");
}
numberOfWorkers = newNumberOfWorkers;
ScaleWorkers(numberOfWorkers);
}
#endregion // ScaleWorkers
#region DrawOSM-Map
if (osmMapRect == null)
{
osmMapRect = new OSMMapRect();
}
osmMapRect.DrawOSMMapRect(new Rect(0, 300, this.position.width, this.position.height - 300));
if (osmMapRect.ShouldRepaint())
{
Repaint();
}
#endregion // DrawOSM-Map
}
private void OnStatusUpdateMessage(StatusUpdateMessage statusUpdateMessage)
{
StatusContent = statusUpdateMessage.Message;
}