private XamDataTree CreateXamDataTree()
{
XamDataTree MyTree = new XamDataTree();
ExperimentNode experiment1 = new ExperimentNode("Experiment1");
CompoundsGroupNode compoundsGroup = new CompoundsGroupNode("CompoundsGroup1");
ClusterNode cluster = new ClusterNode("Cluster1");
cluster.AddChild(new ExperimentNode("Experiment2"));
ExperimentNode experiment3 = new ExperimentNode("Experiment3");
experiment3.HoverText = "My experiment";
experiment3.AddChild(new CompoundsGroupNode("CompoundsGroup2"));
experiment3.AddChild(new CompoundsGroupNode("CompoundsGroup3"));
cluster.AddChild(experiment3);
experiment1.AddChild(compoundsGroup);
experiment1.AddChild(cluster);
MyTree.ItemsSource = new ObservableCollection <IDataNode> {
experiment1
};
NodeLayout mylayout = new NodeLayout();
mylayout.Key = "NodeLayout1";
mylayout.TargetTypeName = "IDataNode";
mylayout.DisplayMemberPath = "Name";
MyTree.GlobalNodeLayouts.Add(mylayout);
return(MyTree);
}
/// <summary>
/// Return an execution back into the domain.
/// </summary>
/// <param name="contextID">The context ID of the execution.</param>
/// <param name="nodeID">The node ID where the execution was run.</param>
/// <param name="result">The serialized result of the excution.</param>
public void ReturnExecution(string contextID, ClusterNode node, byte[] result)
{
SerializationEngine serializer = new SerializationEngine ();
PrestoResult resultObj = (PrestoResult)serializer.Deserialize(result);
resultObj.ExecutionNode = node;
Cluster.ReturnExecution(contextID, resultObj);
}
public Task <int> UpdateClusterNodeAsync(ClusterNode entity)
{
serviceDbContext.Entry(entity).State = EntityState.Modified;
return(serviceDbContext.SaveChangesAsync());
}
internal void Resume(int suspended)
{
Assert.IsNull(members[suspended]);
members[suspended] = new ClusterNode(cfg, cfg.Members[suspended], new Address(basePort + suspended), GetAddressOf);
members[suspended].Attachment = attachments[suspended];
}
//public IEnumerator agglomerate()
public void agglomerate(ProgressUpdateDelegate progFunc)
{
if (items.Count <= 1)
{
clusters = new ClusterNode[0];
return;
//yield break;
}
clusters = new ClusterNode[items.Count * 2 - 1];
for (int i = 0; i < items.Count; i++)
{
clusters[i] = new ClusterNode(items[i], i);
}
float[][] distances = new float[items.Count * 2 - 1][];
for (int i = 0; i < distances.Length; i++)
{
distances[i] = new float[items.Count * 2 - 1];
}
int numClussters = items.Count;
List <ClusterNode> unclustered = new List <ClusterNode>();
for (int i = 0; i < numClussters; i++)
{
unclustered.Add(clusters[i]);
for (int j = 0; j < numClussters; j++)
{
distances[i][j] = euclidean_distance(clusters[i].centroid, clusters[j].centroid);
}
}
int height = 0;
System.Timers.Timer timer = new System.Timers.Timer();
timer.Start();
while (unclustered.Count > 1)
{
height++;
//find closest pair
float min = 10e15f;
int a, b;
a = b = -1;
for (int i = 0; i < unclustered.Count; i++)
{
for (int j = 0; j < unclustered.Count; j++)
{
if (i != j)
{
int idxa = unclustered[i].idx;
int idxb = unclustered[j].idx;
float newDist = distances[idxa][idxb];
if (newDist < min)
{
min = newDist;
a = idxa;
b = idxb;
}
}
}
}
//make a new cluster with pair as children set merge height
numClussters++;
ClusterNode cn = new ClusterNode(clusters[a], clusters[b], numClussters - 1, height, min, clusters);
//remove children from unclustered
unclustered.Remove(clusters[a]);
unclustered.Remove(clusters[b]);
//add new cluster to unclustered
clusters[numClussters - 1] = cn;
unclustered.Add(cn);
//update new clusteres distance
for (int i = 0; i < numClussters - 1; i++)
{
distances[numClussters - 1][i] = euclidean_distance(clusters[numClussters - 1].centroid, clusters[i].centroid);
distances[i][numClussters - 1] = euclidean_distance(clusters[i].centroid, clusters[numClussters - 1].centroid);
}
//if (timer.Interval > .2f)
//{
// yield return null;
// timer.Start();
//}
if (progFunc != null)
{
progFunc("Creating clusters:", ((float)(items.Count - unclustered.Count)) / items.Count);
}
}
}
private static void InitClusterNode(ClusterNode node)
{
node.RootActors = new List<IActorRef>();
foreach (var role in node.Roles)
{
IActorRef rootActor;
switch (role)
{
case "table":
rootActor = node.Context.System.ActorOf(Props.Create(
() => new Table("TestTable", node.Context.ClusterActorDiscovery,
typeof(IncrementalIntegerIdGenerator), null)));
break;
case "container":
rootActor = node.Context.System.ActorOf(Props.Create(
() => new TableContainer("TestTable", node.Context.ClusterActorDiscovery,
typeof(CommonActorFactory<TestActor>), null, null)));
break;
default:
throw new InvalidOperationException("Invalid role: " + role);
}
node.RootActors.Add(rootActor);
}
}
public void constructClustersOfBrokenGateways()
{
foreach (var gateway in _dataStorage.gateways)
{
foreach (var inbound in gateway.inbound_gateways)
{
var clusterName = inbound.Key;
foreach (var server in inbound.Value)
{
if (!_dataStorage.foundServers.Contains(server.connection.name))
{
var clusterToAddTo = _dataStorage.errorClusters.Where(c => c.name == clusterName)
.Select(c => c).FirstOrDefault();
var node = new ServerNode
{
server_id = server.connection.name,
server_name = "Unknown name",
ntv_error = true,
clients = new ConcurrentBag <ConnectionNode>()
};
if (clusterToAddTo is null)
{
var newCluster = new ClusterNode
{
name = clusterName,
servers = new ConcurrentBag <ServerNode>()
};
newCluster.servers.Add(node);
_dataStorage.errorClusters.Add(newCluster);
}
else
{
clusterToAddTo.servers.Add(node);
}
_dataStorage.processedServers.Add(node);
_dataStorage.foundServers.Add(server.connection.name);
}
}
}
foreach (var outbound in gateway.outbound_gateways)
{
var clusterName = outbound.Key;
if (!_dataStorage.foundServers.Contains(outbound.Value.connection.name))
{
var clusterToAddTo = _dataStorage.errorClusters.Where(c => c.name == clusterName).Select(c => c)
.FirstOrDefault();
var node = new ServerNode
{
server_id = outbound.Value.connection.name,
server_name = "Unknown name",
ntv_error = true,
clients = new ConcurrentBag <ConnectionNode>()
};
if (clusterToAddTo is null)
{
var newCluster = new ClusterNode
{
name = clusterName,
servers = new ConcurrentBag <ServerNode>()
};
newCluster.servers.Add(node);
_dataStorage.errorClusters.Add(newCluster);
}
else
{
clusterToAddTo.servers.Add(node);
}
_dataStorage.processedServers.Add(node);
_dataStorage.foundServers.Add(outbound.Value.connection.name);
}
}
foreach (var cluster in _dataStorage.processedClusters)
{
if (!cluster.ContainsServer(gateway.server_id))
{
continue;
}
if (gateway.name is null)
{
continue;
}
cluster.name = gateway.name;
}
}
foreach (var cluster in _dataStorage.errorClusters)
{
_dataStorage.processedClusters.Add(cluster);
}
}
public async Task <bool> UpdateClusterNode(ClusterNode node)
{
return(await _db.HashSetAsync(_clusterOptions.ClusterID, node.Address, JsonConvert.SerializeObject(node)));
}
public virtual ClusterNode <T> Merge(ClusterNode <T> cont)
{
return(Merge(cont.Content));
}
public virtual bool Contains(ClusterNode <T> cont)
{
return(Contains(cont.Content));
}
public virtual float BreakDraw(ClusterNode <T> cont)
{
return(BreakDraw(cont.Content));
}
public bool agglomerate(ProgressUpdateCancelableDelegate progFunc)
{
wasCanceled = true;
if (progFunc != null)
{
wasCanceled = progFunc("Filling Priority Queue:", 0);
}
if (items.Count <= 1)
{
clusters = new ClusterNode[0];
return(false);
//yield break;
}
clusters = new ClusterNode[items.Count * 2 - 1];
for (int i = 0; i < items.Count; i++)
{
clusters[i] = new ClusterNode(items[i], i);
}
int numClussters = items.Count;
List <ClusterNode> unclustered = new List <ClusterNode>();
for (int i = 0; i < numClussters; i++)
{
clusters[i].isUnclustered = true;
unclustered.Add(clusters[i]);
}
int height = 0;
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
timer.Start();
float largestDistInQ = 0;
long usedMemory = GC.GetTotalMemory(false) / 1000000;
PriorityQueue <float, ClusterDistance> pq = new PriorityQueue <float, ClusterDistance>();
//largestDistInQ = _RefillPriorityQWithSome(pq, unclustered, clusters /*,null,null*/);
int numRefills = 0;
while (unclustered.Count > 1)
{
int numToFindClosetPair = 0;
height++;
//find closest pair
if (pq.Count == 0)
{
numRefills++;
usedMemory = GC.GetTotalMemory(false) / 1000000;
if (progFunc != null)
{
wasCanceled = progFunc("Refilling Q:" + ((float)(items.Count - unclustered.Count) * 100) / items.Count + " unclustered:" + unclustered.Count + " inQ:" + pq.Count + " usedMem:" + usedMemory,
((float)(items.Count - unclustered.Count)) / items.Count);
}
largestDistInQ = _RefillPriorityQWithSome(pq, unclustered, clusters, progFunc);
if (pq.Count == 0)
{
break;
}
}
KeyValuePair <float, ClusterDistance> closestPair = pq.Dequeue();
// should only consider unclustered pairs. It is more effecient to discard nodes that have already been clustered as they are popped off the Q
// than to try to remove them from the Q when they have been clustered.
while (!closestPair.Value.a.isUnclustered || !closestPair.Value.b.isUnclustered)
{
if (pq.Count == 0)
{
numRefills++;
usedMemory = GC.GetTotalMemory(false) / 1000000;
if (progFunc != null)
{
wasCanceled = progFunc("Creating clusters:" + ((float)(items.Count - unclustered.Count) * 100) / items.Count + " unclustered:" + unclustered.Count + " inQ:" + pq.Count + " usedMem:" + usedMemory,
((float)(items.Count - unclustered.Count)) / items.Count);
}
largestDistInQ = _RefillPriorityQWithSome(pq, unclustered, clusters, progFunc);
if (pq.Count == 0)
{
break;
}
}
closestPair = pq.Dequeue();
numToFindClosetPair++;
}
//make a new cluster with pair as children set merge height
numClussters++;
ClusterNode cn = new ClusterNode(closestPair.Value.a, closestPair.Value.b, numClussters - 1, height, closestPair.Key, clusters);
//remove children from unclustered
unclustered.Remove(closestPair.Value.a);
unclustered.Remove(closestPair.Value.b);
//We NEED TO REMOVE ALL DISTANCE PAIRS THAT INVOLVE A AND B FROM PRIORITY Q. However searching for all these pairs and removing is very slow.
// Instead we will leave them in the Queue and flag the clusters as isUnclustered = false and discard them as they are popped from the Q which is O(1) operation.
closestPair.Value.a.isUnclustered = false;
closestPair.Value.b.isUnclustered = false;
//add new cluster to unclustered
int newIdx = numClussters - 1;
if (newIdx == clusters.Length)
{
Debug.LogError("how did this happen");
}
clusters[newIdx] = cn;
unclustered.Add(cn);
cn.isUnclustered = true;
//update new clusteres distance
for (int i = 0; i < unclustered.Count - 1; i++)
{
float dist = euclidean_distance(cn.centroid, unclustered[i].centroid);
if (dist < largestDistInQ) //avoid cluttering Qwith
{
pq.Add(new KeyValuePair <float, ClusterDistance>(dist, new ClusterDistance(cn, unclustered[i])));
}
}
//if (timer.Interval > .2f)
//{
// yield return null;
// timer.Start();
//}
if (wasCanceled)
{
break;
}
usedMemory = GC.GetTotalMemory(false) / 1000000;
if (progFunc != null)
{
wasCanceled = progFunc("Creating clusters:" + ((float)(items.Count - unclustered.Count) * 100) / items.Count + " unclustered:" + unclustered.Count + " inQ:" + pq.Count + " usedMem:" + usedMemory,
((float)(items.Count - unclustered.Count)) / items.Count);
}
}
if (progFunc != null)
{
wasCanceled = progFunc("Finished clustering:", 100);
}
//Debug.Log("Time " + timer.Elapsed);
if (wasCanceled)
{
return(false);
}
else
{
return(true);
}
}
public ClusterDistance(ClusterNode aa, ClusterNode bb)
{
a = aa;
b = bb;
}
/// <summary>
/// Deliver the message sent from an outside node.
/// </summary>
/// <param name="payload">The message sent from the outside node.</param>
/// <param name="sender"> The node object of the sending Node.</param>
public void DeliverMessage(string payload, ClusterNode sender)
{
Cluster.DeliverMessage(payload, sender);
}
private static void InitClusterNode(ClusterNode node)
{
node.RootActors = new List<IActorRef>();
foreach (var role in node.Roles)
{
IActorRef rootActor;
switch (role)
{
case "producer":
rootActor = node.Context.System.ActorOf(Props.Create(() => new EchoProducerActor(node.Context)));
break;
case "consumer":
rootActor = node.Context.System.ActorOf(Props.Create(() => new EchoConsumerActor(node.Context)));
break;
default:
throw new InvalidOperationException("Invalid role: " + role);
}
node.RootActors.Add(rootActor);
}
}
public async Task <bool> RemoveClusterNode(ClusterNode node)
{
return(await _db.HashDeleteAsync(_clusterOptions.ClusterID, node.Address));
}
public virtual int MergeSize(ClusterNode <T> cont)
{
return(MergeSize(cont.Content));
}
public virtual int Distance(ClusterNode <T> cont)
{
return(Distance(cont.Content));
}
static void Main(string[] args)
{
while (localPort < 1 || remotePort < 1)
{
Console.Write("Local port : ");
localPort = Convert.ToInt32(Console.ReadLine());
Console.Write("Remote port : ");
remotePort = Convert.ToInt32(Console.ReadLine());
}
n = new ClusterNode("127.0.0.1", remotePort, localPort, ClusterHealthy, ClusterUnhealthy, MessageReceived, false);
bool runForever = true;
while (runForever)
{
Console.Write("Command [? for help]: ");
string userInput = Console.ReadLine();
if (String.IsNullOrEmpty(userInput)) continue;
switch (userInput)
{
case "?":
Console.WriteLine("---");
Console.WriteLine(" q quit");
Console.WriteLine(" ? this menu");
Console.WriteLine(" cls clear screen");
Console.WriteLine(" send send message to peer");
Console.WriteLine(" sendasync send message to peer, asynchronously");
Console.WriteLine(" health display cluster health");
break;
case "q":
runForever = false;
break;
case "cls":
Console.Clear();
break;
case "send":
Console.Write("Data: ");
userInput = Console.ReadLine();
if (Send(Encoding.UTF8.GetBytes(userInput)))
{
Console.WriteLine("Success");
}
else
{
Console.WriteLine("Failed");
}
break;
case "sendasync":
Console.Write("Data: ");
userInput = Console.ReadLine();
if (SendAsync(Encoding.UTF8.GetBytes(userInput)))
{
Console.WriteLine("Success");
}
else
{
Console.WriteLine("Failed");
}
break;
case "health":
Console.WriteLine("Healthy: " + n.IsHealthy());
break;
}
}
}
private static void CreateClusterNode(Config commonConfig, int port, params string[] roles)
{
var config = commonConfig
.WithFallback("akka.remote.helios.tcp.port = " + port)
.WithFallback("akka.cluster.roles = " + "[" + string.Join(",", roles) + "]");
var system = ActorSystem.Create("BasicCluster", config);
var cluster = Cluster.Get(system);
var clusterNode = new ClusterNode
{
Context = new ClusterNodeContext
{
System = system,
ClusterActorDiscovery = system.ActorOf(Props.Create(() => new ClusterActorDiscovery(cluster)),
"cluster_actor_discovery")
},
Roles = roles
};
InitClusterNode(clusterNode);
_clusterNodes.Add(clusterNode);
}
/// <summary>
/// Deliver the message sent from an outside node.
/// </summary>
/// <param name="payload">The message sent from the outside node.</param>
/// <param name="sender"> The node object of the sending Node.</param>
public void DeliverMessage(string payload, ClusterNode sender)
{
Cluster.DeliverMessage(payload, sender);
}
static void Main(string[] args)
{
while (localPort < 1 || remotePort < 1)
{
Console.Write("Local port : ");
localPort = Convert.ToInt32(Console.ReadLine());
Console.Write("Remote port : ");
remotePort = Convert.ToInt32(Console.ReadLine());
}
n = new ClusterNode("127.0.0.1", localPort, "127.0.0.1", remotePort, null, null);
n.AcceptInvalidCertificates = true;
n.MutuallyAuthenticate = true;
n.Debug = false;
n.ReadDataStream = false;
n.PresharedKey = "0000000000000000";
n.MessageReceived = MessageReceived;
n.StreamReceived = StreamReceived;
n.ClusterHealthy = ClusterHealthy;
n.ClusterUnhealthy = ClusterUnhealthy;
n.Start();
bool runForever = true;
while (runForever)
{
Console.Write("Command [? for help]: ");
string userInput = Console.ReadLine();
if (String.IsNullOrEmpty(userInput))
{
continue;
}
byte[] data = null;
MemoryStream ms = null;
switch (userInput)
{
case "?":
Console.WriteLine("---");
Console.WriteLine(" q quit");
Console.WriteLine(" ? this menu");
Console.WriteLine(" cls clear screen");
Console.WriteLine(" send bytes send message to peer");
Console.WriteLine(" send bytes async send message to peer, asynchronously");
Console.WriteLine(" send stream send message to peer using a stream");
Console.WriteLine(" send stream async send message to peer using a stream, asynchronously");
Console.WriteLine(" health display cluster health");
break;
case "q":
runForever = false;
break;
case "cls":
Console.Clear();
break;
case "send bytes":
Console.Write("Data: ");
userInput = Console.ReadLine();
if (n.Send(Encoding.UTF8.GetBytes(userInput)))
{
Console.WriteLine("Success");
}
else
{
Console.WriteLine("Failed");
}
break;
case "send bytes async":
Console.Write("Data: ");
userInput = Console.ReadLine();
if (n.SendAsync(Encoding.UTF8.GetBytes(userInput)).Result)
{
Console.WriteLine("Success");
}
else
{
Console.WriteLine("Failed");
}
break;
case "send stream":
Console.Write("Data: ");
userInput = Console.ReadLine();
data = Encoding.UTF8.GetBytes(userInput);
ms = new MemoryStream(data);
if (n.Send(data.Length, ms))
{
Console.WriteLine("Success");
}
else
{
Console.WriteLine("Failed");
}
break;
case "send stream async":
Console.Write("Data: ");
userInput = Console.ReadLine();
data = Encoding.UTF8.GetBytes(userInput);
ms = new MemoryStream(data);
if (n.SendAsync(data.Length, ms).Result)
{
Console.WriteLine("Success");
}
else
{
Console.WriteLine("Failed");
}
break;
case "health":
Console.WriteLine("Healthy: " + n.IsHealthy);
break;
}
}
}
public void Node(string nodeAddress)
{
_nodes.Add(ClusterNode.Parse(nodeAddress));
}
private RedisCluster(ClusterNode[] nodes)
{
this.nodes = new List<ClusterNode>(nodes);
}
Task IBucketInternal.Bootstrap(ClusterNode clusterNode)
{
throw new NotImplementedException();
}
public Task <int> InsertClusterNodeAsync(ClusterNode entity)
{
serviceDbContext.ClusterNode.Add(entity);
return(serviceDbContext.SaveChangesAsync());
}
public Task <int> DeleteAsync(ClusterNode entity)
{
return(UpdateAsync(entity));
}
protected bool Equals(ClusterNode other)
{
return(string.Equals(ClusterTag, other.ClusterTag));
}
private async Task <List <ClusterNode> > BuildClustersAsync(List <Node> nodes, IDistanceMetric distanceMetric, ProgressData progressData)
{
var nodeCount = nodes
.SelectMany(node => node.NeighborsByDistance.Select(neighbor => neighbor.Node.FirstLeaf.Index))
.Concat(nodes.Select(node => node.FirstLeaf.Index))
.Distinct().Count();
progressData.Reset($"Building clusters for {nodeCount} matches...", nodes.Count - 1);
await Task.Run(async() =>
{
while (nodes.Count > 1)
{
// This is a little verbose, but optimized for performance -- O(N) overall.
Node secondNode = null;
Neighbor neighborToCluster = new Neighbor()
{
DistanceSquared = float.MaxValue
};
foreach (var node in nodes)
{
if (node.FirstLeaf.NeighborsByDistance.Count > 0 && node.FirstLeaf.NeighborsByDistance.First().DistanceSquared < neighborToCluster.DistanceSquared)
{
secondNode = node;
neighborToCluster = node.FirstLeaf.NeighborsByDistance.First();
}
if (node.FirstLeaf != node.SecondLeaf && node.SecondLeaf.NeighborsByDistance.Count > 0 && node.SecondLeaf.NeighborsByDistance.First().DistanceSquared < neighborToCluster.DistanceSquared)
{
secondNode = node;
neighborToCluster = node.SecondLeaf.NeighborsByDistance.First();
}
}
ClusterNode clusterNode;
if (secondNode == null)
{
var nodesLargestFirst = nodes.OrderByDescending(node => node.GetOrderedLeafNodes().Count()).Take(2).ToList();
clusterNode = new ClusterNode(nodesLargestFirst[0], nodesLargestFirst[1], double.PositiveInfinity, distanceMetric);
}
else
{
var firstNode = neighborToCluster.Node;
var first = firstNode.GetHighestParent();
var second = secondNode.GetHighestParent();
clusterNode = new ClusterNode(first, second, neighborToCluster.DistanceSquared, distanceMetric);
}
if (clusterNode.First.FirstLeaf != clusterNode.First.SecondLeaf)
{
var removeNeighborsTasks = nodes.Select(node => Task.Run(() =>
{
node.FirstLeaf.NeighborsByDistance?.RemoveAll(neighbor => neighbor.Node == clusterNode.First.SecondLeaf);
if (node.FirstLeaf != node.SecondLeaf)
{
node.SecondLeaf.NeighborsByDistance?.RemoveAll(neighbor => neighbor.Node == clusterNode.First.SecondLeaf);
}
}));
await Task.WhenAll(removeNeighborsTasks);
}
if (clusterNode.Second.FirstLeaf != clusterNode.Second.SecondLeaf)
{
var removeNeighborsTasks = nodes.Select(node => Task.Run(() =>
{
node.FirstLeaf.NeighborsByDistance?.RemoveAll(neighbor => neighbor.Node == clusterNode.Second.FirstLeaf);
if (node.FirstLeaf != node.SecondLeaf)
{
node.SecondLeaf.NeighborsByDistance?.RemoveAll(neighbor => neighbor.Node == clusterNode.Second.FirstLeaf);
}
}));
await Task.WhenAll(removeNeighborsTasks);
}
nodes.Remove(clusterNode.First);
nodes.Remove(clusterNode.Second);
var leafNodes = new HashSet <LeafNode>(clusterNode.GetOrderedLeafNodes());
clusterNode.FirstLeaf.NeighborsByDistance.RemoveAll(neighbor => leafNodes.Contains(neighbor.Node));
clusterNode.SecondLeaf.NeighborsByDistance.RemoveAll(neighbor => leafNodes.Contains(neighbor.Node));
nodes.Add(clusterNode);
progressData.Increment();
}
});
progressData.Reset("Done");
return(nodes.OfType <ClusterNode>().ToList());
}
