/**
* Generates tree for bounded broadcast. Algorithm works as follows:
* The goal is to broadcast to all nodes in range (start, end).
* Given a range (a, b), determine all connections that belong to this range.
* Let the left connections be l_1, l_2, ..... l_n.
* Let the right connections be r_1, r_2, ... , r_n.
* To left connection l_i assign the range [b_{i-1}, b_i).
* To right connection r_i assign the range [r_i, r_{i-1}]
* To the connection ln assign range [l_{n-1}, end)
* To the connection rn assign range (start, r_{n-1}]
*/
public override void GenerateTree(Channel q, MapReduceArgs mr_args)
{
ArrayList gen_list = mr_args.GenArg as ArrayList;
string start_range = gen_list[0] as string;
AHAddress start_addr = (AHAddress)AddressParser.Parse(start_range);
AHAddress end_addr;
string end_range;
/// If users do not specify an end range, this method understands
/// that users intend to broadcasting the whole range.
/// Thus, the address of end range is set to (start_address - 2),
/// the farthest address from the start_addr.
if (gen_list.Count < 2)
{
BigInteger start_int = start_addr.ToBigInteger();
BigInteger end_int = start_int - 2;
end_addr = new AHAddress(end_int);
end_range = end_addr.ToString();
}
else
{
end_range = gen_list[1] as string;
end_addr = (AHAddress)AddressParser.Parse(end_range);
}
Log("generating child tree, range start: {0}, range end: {1}.", start_range, end_range);
//we are at the start node, here we go:
ConnectionTable tab = _node.ConnectionTable;
ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
List <MapReduceInfo> retval = new List <MapReduceInfo>();
if (InRange(_this_addr, start_addr, end_addr))
{
if (structs.Count > 0)
{
//make connection list in the range.
//left connection list is a list of neighbors which are in the range (this node, end of range)
//right connection list is a list of neighbors which are in the range (start of range, this node)
Brunet.Collections.Pair <List <Connection>, List <Connection> > cons = GetConnectionInfo(_this_addr, start_addr, end_addr, structs);
List <Connection> left_cons = cons.First as List <Connection>;
List <Connection> right_cons = cons.Second as List <Connection>;
//PrintConnectionList(left_cons);
//PrintConnectionList(right_cons);
retval = GenerateTreeInRange(start_addr, end_addr, left_cons, true, mr_args);
List <MapReduceInfo> ret_right = GenerateTreeInRange(start_addr, end_addr, right_cons, false, mr_args);
retval.AddRange(ret_right);
}
else //this node is a leaf node.
//MapReduceInfo mr_info = null;
//retval.Add(mr_info);
//Console.WriteLine("no connection in the range: return null info");
{
}
}
else // _node is out of range. Just pass it to the closest to the middle of range.
{
retval = GenerateTreeOutRange(start_addr, end_addr, mr_args);
}
q.Enqueue(retval.ToArray());
}
// adds a disconnected pair to the pool
public void AddDisconnectedPair(out Address address1, out Address address2, bool nctunnel)
{
address1 = new AHAddress(new RNGCryptoServiceProvider());
byte[] addrbuff = Address.ConvertToAddressBuffer(address1.ToBigInteger() + (Address.Full / 2));
Address.SetClass(addrbuff, AHAddress._class);
address2 = new AHAddress(addrbuff);
AddDisconnectedPair(address1, address2, nctunnel);
}
/// <summary>Calculate the Left AHAddress of the given address.</summary>
public AHAddress GetLeftNearTarget(AHAddress address)
{
BigInteger local_int_add = address.ToBigInteger();
//must have even addresses so increment twice
local_int_add -= 2;
//Make sure we don't overflow:
BigInteger tbi = new BigInteger(local_int_add % Address.Full);
return(new AHAddress(tbi));
}
public void Test()
{
ITunnelOverlap _ito = new SimpleTunnelOverlap();
Address addr_x = new AHAddress(new RNGCryptoServiceProvider());
byte[] addrbuff = Address.ConvertToAddressBuffer(addr_x.ToBigInteger() + (Address.Full / 2));
Address.SetClass(addrbuff, AHAddress._class);
Address addr_y = new AHAddress(addrbuff);
ArrayList addresses = new ArrayList();
ConnectionTable ct_x = new ConnectionTable();
ConnectionTable ct_y = new ConnectionTable();
ConnectionTable ct_empty = new ConnectionTable();
for (int i = 1; i <= 10; i++)
{
addrbuff = Address.ConvertToAddressBuffer(addr_x.ToBigInteger() + (i * Address.Full / 16));
Address.SetClass(addrbuff, AHAddress._class);
addresses.Add(new AHAddress(addrbuff));
TransportAddress ta = TransportAddressFactory.CreateInstance("brunet.tcp://158.7.0.1:5000");
Edge fe = new FakeEdge(ta, ta, TransportAddress.TAType.Tcp);
ct_x.Add(new Connection(fe, addresses[i - 1] as AHAddress, "structured", null, null));
if (i == 10)
{
ct_y.Add(new Connection(fe, addresses[i - 1] as AHAddress, "structured", null, null));
}
}
ConnectionType con_type = ConnectionType.Structured;
IDictionary id = _ito.GetSyncMessage(null, addr_x, ct_x.GetConnections(con_type));
Assert.AreEqual(_ito.EvaluateOverlap(ct_y.GetConnections(con_type), id)[0].Address, addresses[9], "Have an overlap!");
Assert.AreEqual(_ito.EvaluateOverlap(ct_empty.GetConnections(con_type), id).Count, 0, "No overlap!");
Assert.AreEqual(addresses.Contains(_ito.EvaluatePotentialOverlap(id)), true, "EvaluatePotentialOverlap returns valid!");
}
public AHAddress GetNearTarget(AHAddress address)
{
/**
* try to get at least one neighbor using forwarding through the
* leaf . The forwarded address is 2 larger than the address of
* the new node that is getting connected.
*/
BigInteger local_int_add = address.ToBigInteger();
//must have even addresses so increment twice
local_int_add += 2;
//Make sure we don't overflow:
BigInteger tbi = new BigInteger(local_int_add % Address.Full);
return(new AHAddress(tbi));
}
/// <summary>Calculates a shortcut using a harmonic distribution as in a
/// Symphony-lke shortcut.</summary>
protected virtual AHAddress ComputeShortcutTarget(AHAddress addr)
{
int network_size = _addrs.Count;
double logN = (double)(Brunet.Address.MemSize * 8);
double logk = Math.Log((double)network_size, 2.0);
double p = _rand.NextDouble();
double ex = logN - (1.0 - p) * logk;
int ex_i = (int)Math.Floor(ex);
double ex_f = ex - Math.Floor(ex);
//Make sure 2^(ex_long+1) will fit in a long:
int ex_long = ex_i % 63;
int ex_big = ex_i - ex_long;
ulong dist_long = (ulong)Math.Pow(2.0, ex_long + ex_f);
//This is 2^(ex_big):
BigInteger big_one = 1;
BigInteger dist_big = big_one << ex_big;
BigInteger rand_dist = dist_big * dist_long;
// Add or subtract random distance to the current address
BigInteger t_add = addr.ToBigInteger();
// Random number that is 0 or 1
if (_rand.Next(2) == 0)
{
t_add += rand_dist;
}
else
{
t_add -= rand_dist;
}
BigInteger target_int = new BigInteger(t_add % Address.Full);
if ((target_int & 1) == 1)
{
target_int -= 1;
}
byte[] buf = Address.ConvertToAddressBuffer(target_int);
Address.SetClass(buf, AHAddress.ClassValue);
return(new AHAddress(buf));
}
/**
* When a node is out of the range, this method is called.
* This method tries to find the nearest node to the middle of range using greedty algorithm.
* return list of MapReduceInfo
*/
private List <MapReduceInfo> GenerateTreeOutRange(AHAddress start, AHAddress end, MapReduceArgs mr_args)
{
List <MapReduceInfo> retval = new List <MapReduceInfo>();
BigInteger up = start.ToBigInteger();
BigInteger down = end.ToBigInteger();
BigInteger mid_range = (up + down) / 2;
if (mid_range % 2 == 1)
{
mid_range = mid_range - 1;
}
AHAddress mid_addr = new AHAddress(mid_range);
//if (!mid_addr.IsBetweenFromLeft(start, end) ) {
if (!InRange(mid_addr, start, end))
{
mid_range += Address.Half;
mid_addr = new AHAddress(mid_range);
}
ArrayList gen_arg = new ArrayList();
if (NextGreedyClosest(mid_addr) != null)
{
AHGreedySender ags = new AHGreedySender(_node, mid_addr);
string start_range = start.ToString();
string end_range = end.ToString();
gen_arg.Add(start_range);
gen_arg.Add(end_range);
MapReduceInfo mr_info = new MapReduceInfo((ISender)ags,
new MapReduceArgs(this.TaskName,
mr_args.MapArg,
gen_arg,
mr_args.ReduceArg));
Log("{0}: {1}, out of range, moving to the closest node to mid_range: {2} to target node, range start: {3}, range end: {4}",
this.TaskName, _node.Address, mid_addr, start, end);
retval.Add(mr_info);
}
else
{
// cannot find a node in the range.
}
return(retval);
}
///This tester simply establishes the Brunet network and log the edges made
static void Main(string[] args)
{
String config_file = args[0];
NetworkConfiguration network_configuration = NetworkConfiguration.Deserialize(config_file);
for(int i = 0; i < network_configuration.Nodes.Count; i++){
NodeConfiguration this_node_configuration = (NodeConfiguration)network_configuration.Nodes[i];
TransportAddressConfiguration local_ta_configuration = (TransportAddressConfiguration)this_node_configuration.TransportAddresses[0];
short port = local_ta_configuration.Port;
SHA1 sha = new SHA1CryptoServiceProvider();
String local_ta = local_ta_configuration.GetTransportAddressURI();
//We take the local transport address plus the port number to be hashed to obtain a random AHAddress
byte[] hashedbytes = sha.ComputeHash(Encoding.UTF8.GetBytes(local_ta + port));
//inforce type 0
hashedbytes[Address.MemSize - 1] &= 0xFE;
AHAddress _local_ahaddress = new AHAddress(hashedbytes);
Console.WriteLine(_local_ahaddress.ToBigInteger().ToString() + ' ' + local_ta_configuration.Address + ' ' + (port%25000));
}
}
public void Test()
{
Address addr_x = new AHAddress(new RNGCryptoServiceProvider());
byte[] addrbuff = Address.ConvertToAddressBuffer(addr_x.ToBigInteger() + (Address.Full / 2));
Address.SetClass(addrbuff, AHAddress._class);
Address addr_y = new AHAddress(addrbuff);
List <Connection> connections = new List <Connection>();
ConnectionTable ct_x = new ConnectionTable();
ConnectionTable ct_y = new ConnectionTable();
ConnectionTable ct_empty = new ConnectionTable();
NCService ncservice = new NCService();
Connection fast_con = null;
for (int i = 1; i <= 11; i++)
{
addrbuff = Address.ConvertToAddressBuffer(addr_x.ToBigInteger() + (i * Address.Full / 16));
Address.SetClass(addrbuff, AHAddress._class);
Address addr = new AHAddress(addrbuff);
Connection con = null;
TransportAddress ta = TransportAddressFactory.CreateInstance("brunet.tcp://158.7.0.1:5000");
Edge fe = new FakeEdge(ta, ta, TransportAddress.TAType.Tcp);
if (i <= 10)
{
con = new Connection(fe, addr, "structured", null, null);
ct_x.Add(con);
if (i % 2 == 0)
{
ncservice.ProcessSample(DateTime.UtcNow, String.Empty, addr,
new Point(new double[] { 0, 0 }, 0), 0, i * 10);
}
}
else
{
fast_con = new Connection(fe, addr, "structured", null, null);
ncservice.ProcessSample(DateTime.UtcNow, String.Empty, addr,
new Point(new double[] { 0, 0 }, 0), 0, 5);
}
if (i == 10)
{
ct_y.Add(con);
}
connections.Add(con);
}
IRelayOverlap sto = new SimpleRelayOverlap();
IRelayOverlap nto = new NCRelayOverlap(ncservice);
ConnectionType con_type = ConnectionType.Structured;
List <Connection> pre_cons = new List <Connection>();
pre_cons.Add(connections[9]);
IDictionary id = nto.GetSyncMessage(pre_cons, addr_x, ct_x.GetConnections(con_type));
// We do have some pre-existing overlap
Assert.AreEqual(nto.EvaluateOverlap(ct_y.GetConnections(con_type), id)[0], connections[9], "NC: Have an overlap!");
Assert.AreEqual(sto.EvaluateOverlap(ct_y.GetConnections(con_type), id)[0], connections[9], "Simple: Have an overlap!");
// We have no overlap with an empty connection table
Assert.AreEqual(nto.EvaluateOverlap(ct_empty.GetConnections(con_type), id).Count, 0, "No overlap!");
Assert.AreEqual(sto.EvaluateOverlap(ct_empty.GetConnections(con_type), id).Count, 0, "No overlap!");
// latency[0] == -1
Assert.AreEqual(connections[1].Address.Equals(nto.EvaluatePotentialOverlap(id)), true,
"NC: EvaluatePotentialOverlap returns expected!");
Assert.AreEqual(ct_x.Contains(con_type, sto.EvaluatePotentialOverlap(id)), true,
"Simple: EvaluatePotentialOverlap returns valid!");
ct_y.Add(fast_con);
ct_x.Add(fast_con);
id = nto.GetSyncMessage(pre_cons, addr_x, ct_x.GetConnections(con_type));
Assert.AreEqual(fast_con.Address.Equals(nto.EvaluatePotentialOverlap(id)), true,
"NC: EvaluatePotentialOverlap returns expected!");
Assert.AreEqual(nto.EvaluateOverlap(ct_y.GetConnections(con_type), id)[0], fast_con, "NC: Have better overlap!");
}
public AHAddress GetNearTarget(AHAddress address)
{
/**
* try to get at least one neighbor using forwarding through the
* leaf . The forwarded address is 2 larger than the address of
* the new node that is getting connected.
*/
BigInteger local_int_add = address.ToBigInteger();
//must have even addresses so increment twice
local_int_add += 2;
//Make sure we don't overflow:
BigInteger tbi = new BigInteger(local_int_add % Address.Full);
return new AHAddress(tbi);
}
/// <summary>Calculates a shortcut using a harmonic distribution as in a
/// Symphony-lke shortcut.</summary>
protected virtual AHAddress ComputeShortcutTarget(AHAddress addr)
{
int network_size = _addrs.Count;
double logN = (double)(Brunet.Address.MemSize * 8);
double logk = Math.Log( (double) network_size, 2.0 );
double p = _rand.NextDouble();
double ex = logN - (1.0 - p)*logk;
int ex_i = (int)Math.Floor(ex);
double ex_f = ex - Math.Floor(ex);
//Make sure 2^(ex_long+1) will fit in a long:
int ex_long = ex_i % 63;
int ex_big = ex_i - ex_long;
ulong dist_long = (ulong)Math.Pow(2.0, ex_long + ex_f);
//This is 2^(ex_big):
BigInteger big_one = 1;
BigInteger dist_big = big_one << ex_big;
BigInteger rand_dist = dist_big * dist_long;
// Add or subtract random distance to the current address
BigInteger t_add = addr.ToBigInteger();
// Random number that is 0 or 1
if( _rand.Next(2) == 0 ) {
t_add += rand_dist;
}
else {
t_add -= rand_dist;
}
BigInteger target_int = new BigInteger(t_add % Address.Full);
if((target_int & 1) == 1) {
target_int -= 1;
}
byte[]buf = Address.ConvertToAddressBuffer(target_int);
Address.SetClass(buf, AHAddress.ClassValue);
return new AHAddress(buf);
}
public void Test()
{
Address addr_x = new AHAddress(new RNGCryptoServiceProvider());
byte[] addrbuff = Address.ConvertToAddressBuffer(addr_x.ToBigInteger() + (Address.Full / 2));
Address.SetClass(addrbuff, AHAddress._class);
Address addr_y = new AHAddress(addrbuff);
List<Connection> connections = new List<Connection>();
ConnectionTable ct_x = new ConnectionTable();
ConnectionTable ct_y = new ConnectionTable();
ConnectionTable ct_empty = new ConnectionTable();
NCService ncservice = new NCService();
Connection fast_con = null;
for(int i = 1; i <= 11; i++) {
addrbuff = Address.ConvertToAddressBuffer(addr_x.ToBigInteger() + (i * Address.Full / 16));
Address.SetClass(addrbuff, AHAddress._class);
Address addr = new AHAddress(addrbuff);
Connection con = null;
TransportAddress ta = TransportAddressFactory.CreateInstance("brunet.tcp://158.7.0.1:5000");
Edge fe = new FakeEdge(ta, ta, TransportAddress.TAType.Tcp);
if(i <= 10) {
con = new Connection(fe, addr, "structured", null, null);
ct_x.Add(con);
if(i % 2 == 0) {
ncservice.ProcessSample(DateTime.UtcNow, String.Empty, addr,
new Point(new double[] {0, 0}, 0), 0, i*10);
}
} else {
fast_con = new Connection(fe, addr, "structured", null, null);
ncservice.ProcessSample(DateTime.UtcNow, String.Empty, addr,
new Point(new double[] {0, 0}, 0), 0, 5);
}
if(i == 10) {
ct_y.Add(con);
}
connections.Add(con);
}
ITunnelOverlap sto = new SimpleTunnelOverlap();
ITunnelOverlap nto = new NCTunnelOverlap(ncservice);
ConnectionType con_type = ConnectionType.Structured;
List<Connection> pre_cons = new List<Connection>();
pre_cons.Add(connections[9]);
IDictionary id = nto.GetSyncMessage(pre_cons, addr_x, ct_x.GetConnections(con_type));
// We do have some pre-existing overlap
Assert.AreEqual(nto.EvaluateOverlap(ct_y.GetConnections(con_type), id)[0], connections[9], "NC: Have an overlap!");
Assert.AreEqual(sto.EvaluateOverlap(ct_y.GetConnections(con_type), id)[0], connections[9], "Simple: Have an overlap!");
// We have no overlap with an empty connection table
Assert.AreEqual(nto.EvaluateOverlap(ct_empty.GetConnections(con_type), id).Count, 0, "No overlap!");
Assert.AreEqual(sto.EvaluateOverlap(ct_empty.GetConnections(con_type), id).Count, 0, "No overlap!");
// latency[0] == -1
Assert.AreEqual(connections[1].Address.Equals(nto.EvaluatePotentialOverlap(id)), true,
"NC: EvaluatePotentialOverlap returns expected!");
Assert.AreEqual(ct_x.Contains(con_type, sto.EvaluatePotentialOverlap(id)), true,
"Simple: EvaluatePotentialOverlap returns valid!");
ct_y.Add(fast_con);
ct_x.Add(fast_con);
id = nto.GetSyncMessage(pre_cons, addr_x, ct_x.GetConnections(con_type));
Assert.AreEqual(fast_con.Address.Equals(nto.EvaluatePotentialOverlap(id)), true,
"NC: EvaluatePotentialOverlap returns expected!");
Assert.AreEqual(nto.EvaluateOverlap(ct_y.GetConnections(con_type), id)[0], fast_con, "NC: Have better overlap!");
}
/**
<summary>This method returns matching querying address from caching address.<\summary>
*/
protected AHAddress AddressTranspose(AHAddress a) {
BigInteger one = new BigInteger(1);
BigInteger s_BIN = one << 80;
BigInteger addr = a.ToBigInteger();
BigInteger addr_j = addr % s_BIN;
BigInteger addr_i = (addr - addr_j) / s_BIN;
BigInteger q = addr_j * s_BIN + addr_i;
byte[] target = Address.ConvertToAddressBuffer(q);
Address.SetClass(target, a.Class);
AHAddress q_addr = new AHAddress(target);
return q_addr;
}
/**
* Generate tree within the range.
* return list of MapReduceInfo
*/
private List <MapReduceInfo> GenerateTreeInRange(AHAddress start, AHAddress end, List <Connection> cons, bool left, MapReduceArgs mr_args)
{
//Divide the range and trigger bounded broadcasting again in divided range starting with neighbor.
//Deivided ranges are (start, n_1), (n_1, n_2), ... , (n_m, end)
AHAddress this_minus2 = new AHAddress(_this_addr.ToBigInteger() - 2);
AHAddress this_plus2 = new AHAddress(_this_addr.ToBigInteger() + 2);
List <MapReduceInfo> retval = new List <MapReduceInfo>();
if (cons.Count != 0) //make sure if connection list is not empty!
{
//con_list is sorted.
AHAddress last;
if (left)
{
last = end;
}
else
{
last = start;
}
string rg_start, rg_end;
//the first element of cons is the nearest.
//Let's start with the farthest neighbor first.
for (int i = (cons.Count - 1); i >= 0; i--)
{
ArrayList gen_arg = new ArrayList();
Connection next_c = cons[i];
AHAddress next_addr = (AHAddress)next_c.Address;
ISender sender = next_c.State.Edge;
if (i == 0) // The last bit
{
if (left)
{
// the left nearest neighbor
rg_start = this_plus2.ToString();
rg_end = last.ToString();
}
else
{
// the right nearest neighbor
rg_start = last.ToString();
rg_end = this_minus2.ToString();
}
}
else
{
if (left) //left connections
{
rg_start = next_addr.ToString();
rg_end = last.ToString();
}
else //right connections
{
rg_start = last.ToString();
rg_end = next_addr.ToString();
}
}
gen_arg.Add(rg_start);
gen_arg.Add(rg_end);
MapReduceInfo mr_info = new MapReduceInfo(sender,
new MapReduceArgs(this.TaskName,
mr_args.MapArg,
gen_arg,
mr_args.ReduceArg));
Log("{0}: {1}, adding address: {2} to sender list, range start: {3}, range end: {4}",
this.TaskName, _node.Address, next_c.Address,
gen_arg[0], gen_arg[1]);
if (left)
{
last = new AHAddress(next_addr.ToBigInteger() - 2);
}
else
{
last = new AHAddress(next_addr.ToBigInteger() + 2);
}
retval.Add(mr_info);
}
}
return(retval);
}
// adds a disconnected pair to the pool
public void AddDisconnectedPair(out Address address1, out Address address2, bool nctunnel)
{
address1 = new AHAddress(new RNGCryptoServiceProvider());
byte[] addrbuff = Address.ConvertToAddressBuffer(address1.ToBigInteger() + (Address.Full / 2));
Address.SetClass(addrbuff, AHAddress._class);
address2 = new AHAddress(addrbuff);
AddDisconnectedPair(address1, address2, nctunnel);
}
public void Test()
{
ITunnelOverlap _ito = new SimpleTunnelOverlap();
Address addr_x = new AHAddress(new RNGCryptoServiceProvider());
byte[] addrbuff = Address.ConvertToAddressBuffer(addr_x.ToBigInteger() + (Address.Full / 2));
Address.SetClass(addrbuff, AHAddress._class);
Address addr_y = new AHAddress(addrbuff);
ArrayList addresses = new ArrayList();
ConnectionTable ct_x = new ConnectionTable();
ConnectionTable ct_y = new ConnectionTable();
ConnectionTable ct_empty = new ConnectionTable();
for(int i = 1; i <= 10; i++) {
addrbuff = Address.ConvertToAddressBuffer(addr_x.ToBigInteger() + (i * Address.Full / 16));
Address.SetClass(addrbuff, AHAddress._class);
addresses.Add(new AHAddress(addrbuff));
TransportAddress ta = TransportAddressFactory.CreateInstance("brunet.tcp://158.7.0.1:5000");
Edge fe = new FakeEdge(ta, ta, TransportAddress.TAType.Tcp);
ct_x.Add(new Connection(fe, addresses[i - 1] as AHAddress, "structured", null, null));
if(i == 10) {
ct_y.Add(new Connection(fe, addresses[i - 1] as AHAddress, "structured", null, null));
}
}
ConnectionType con_type = ConnectionType.Structured;
IDictionary id = _ito.GetSyncMessage(null, addr_x, ct_x.GetConnections(con_type));
Assert.AreEqual(_ito.EvaluateOverlap(ct_y.GetConnections(con_type), id)[0].Address, addresses[9], "Have an overlap!");
Assert.AreEqual(_ito.EvaluateOverlap(ct_empty.GetConnections(con_type), id).Count, 0, "No overlap!");
Assert.AreEqual(addresses.Contains(_ito.EvaluatePotentialOverlap(id)), true, "EvaluatePotentialOverlap returns valid!");
}