public override MapReduceInfo[] GenerateTree(MapReduceArgs mr_args) {
object gen_arg = mr_args.GenArg;
Log("{0}: {1}, greedy generator called, arg: {2}.",
this.TaskName, _node.Address, gen_arg);
string address = gen_arg as string;
AHAddress a = (AHAddress) AddressParser.Parse(address);
ArrayList retval = new ArrayList();
ConnectionTable tab = _node.ConnectionTable;
ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
Connection next_closest = structs.GetNearestTo((AHAddress) _node.Address, a);
if (next_closest != null) {
int timeout = -1;
if (mr_args.MillisecTimeout >= 0) { //only if it is a valid timeout
timeout = (int) (TIMEOUT_FRACTION*mr_args.MillisecTimeout);
}
MapReduceInfo mr_info = new MapReduceInfo( (ISender) next_closest.Edge,
new MapReduceArgs(this.TaskName,//arguments do not change much
mr_args.MapArg,
mr_args.GenArg,
mr_args.ReduceArg,
timeout));//only timeout changes
retval.Add(mr_info);
}
Log("{0}: {1}, greedy generator returning: {2} senders.",
this.TaskName, _node.Address, retval.Count);
return (MapReduceInfo[]) retval.ToArray(typeof(MapReduceInfo));
}
public override MapReduceInfo[] GenerateTree(MapReduceArgs mr_args) {
object gen_arg = mr_args.GenArg;
Log("{0}: {1}, greedy generator called, arg: {2}.",
this.TaskName, _node.Address, gen_arg);
string address = gen_arg as string;
AHAddress a = (AHAddress) AddressParser.Parse(address);
ArrayList retval = new ArrayList();
ConnectionTable tab = _node.ConnectionTable;
ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
Connection next_closest = structs.GetNearestTo((AHAddress) _node.Address, a);
if (next_closest != null) {
MapReduceInfo mr_info = new MapReduceInfo( (ISender) next_closest.Edge,
mr_args); //arguments do not change at all
retval.Add(mr_info);
}
Log("{0}: {1}, greedy generator returning: {2} senders.",
this.TaskName, _node.Address, retval.Count);
return (MapReduceInfo[]) retval.ToArray(typeof(MapReduceInfo));
}
/*
* This cannot change our Reducing state.
*/
public State UpdateTree(MapReduceInfo[] tree) {
if( Tree != null ) {
throw new Exception("MapReduce: Tree already set");
}
bool done = Done;
if( ReduceResult != DEFAULT_OBJ ) {
/*
* We have already reduced the map result, we are done, ONLY if
* the tree is empty:
*/
done = done || (tree.Length == 0);
}
else {
//We haven't yet reduced the map result
//Done should be false
}
return new State(MapResult, tree, ChildReductions, ReduceResult, done, Pending, Reducing);
}
protected State(object map, MapReduceInfo[] tree, int cred, object reduce, bool done, ImmutableList<RpcResult> pend, bool reducing) {
MapResult = map;
Tree = tree;
ChildReductions = cred;
ReduceResult = reduce;
Done = done;
Pending = pend;
Reducing = reducing;
}
/** Starts the computation. */
public void Start() {
//invoke map
try {
_map_result = _mr_task.Map(_mr_args.MapArg);
}
catch(Exception x) {
if (ProtocolLog.MapReduce.Enabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("MapReduce: {0}, map exception: {1}.", _node.Address, x));
}
_finished = true;
SendResult(x);
return;
}
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("MapReduce: {0}, map result: {1}.", _node.Address, _map_result));
}
//do an initial reduction and see if we can terminate
try {
bool done; //out parameter
_reduce_result = _mr_task.Reduce(_mr_args.ReduceArg, null, new RpcResult(null, _map_result), out done);
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("MapReduce: {0}, initial reduce result: {1}.", _node.Address, _reduce_result));
}
if (done) {
_finished = true;
SendResult(_reduce_result);
return;
}
} catch(Exception x) {
if (ProtocolLog.MapReduce.Enabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("MapReduce: {0}, initial reduce exception: {1}.", _node.Address, x));
}
_finished = true;
SendResult(x);
return;
}
//compute the list of child targets
MapReduceInfo[] child_mr_info = null;
try {
child_mr_info = _mr_task.GenerateTree(_mr_args);
} catch (Exception x) {
if (ProtocolLog.MapReduce.Enabled) {
child_mr_info = new MapReduceInfo[0];
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("MapReduce: {0}, generate tree exception: {1}.", _node.Address, x));
}
}
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("MapReduce: {0}, child senders count: {1}.", _node.Address, child_mr_info.Length));
}
if (child_mr_info.Length > 0) {
foreach ( MapReduceInfo mr_info in child_mr_info) {
Channel child_q = new Channel(1);
//so far this is thread-safe
_queue_to_child[child_q] = mr_info;
}
foreach (DictionaryEntry de in _queue_to_child) {
Channel child_q = (Channel) de.Key;
MapReduceInfo mr_info = (MapReduceInfo) de.Value;
//the following will prevent the current object from going out of scope.
child_q.EnqueueEvent += new EventHandler(ChildCallback);
try {
_rpc.Invoke(mr_info.Sender, child_q, "mapreduce.Start", mr_info.Args.ToHashtable());
} catch(Exception) {
ChildCallback(child_q, null);
}
}
} else {
// did not generate any child computations, return rightaway
_finished = true;
SendResult(_reduce_result);
return;
}
}
/**
* When a node is out of the range, this method is called.
* This method tries to find the nearest node to the middle of range using greedty algorithm.
* return list of MapReduceInfo
*/
private ArrayList GenerateTreeOutRange(AHAddress start, AHAddress end, MapReduceArgs mr_args, int timeout) {
ArrayList retval = new ArrayList();
BigInteger up = start.ToBigInteger();
BigInteger down = end.ToBigInteger();
BigInteger mid_range = (up + down) /2;
if (mid_range % 2 == 1) {mid_range = mid_range -1; }
AHAddress mid_addr = new AHAddress(mid_range);
if (!mid_addr.IsBetweenFromLeft(start, end) ) {
mid_range += Address.Half;
mid_addr = new AHAddress(mid_range);
}
ArrayList gen_arg = new ArrayList();
if (NextGreedyClosest(mid_addr) != null ) {
AHGreedySender ags = new AHGreedySender(_node, mid_addr);
string start_range = start.ToString();
string end_range = end.ToString();
gen_arg.Add(start_range);
gen_arg.Add(end_range);
MapReduceInfo mr_info = new MapReduceInfo( (ISender) ags,
new MapReduceArgs(this.TaskName,
mr_args.MapArg,
gen_arg,
mr_args.ReduceArg,
timeout));
Log("{0}: {1}, out of range, moving to the closest node to mid_range: {2} to target node, range start: {3}, range end: {4}",
this.TaskName, _node.Address, mid_addr, start, end);
retval.Add(mr_info);
}
else {
// cannot find a node in the range.
}
return retval;
}
/**
* Generate tree within the range.
* return list of MapReduceInfo
*/
private ArrayList GenerateTreeInRange(AHAddress this_addr, AHAddress start, AHAddress end, List<Connection> cons, bool left, MapReduceArgs mr_args, int timeout) {
//Divide the range and trigger bounded broadcasting again in divided range starting with neighbor.
//Deivided ranges are (start, n_1), (n_1, n_2), ... , (n_m, end)
ArrayList retval = new ArrayList();
if (cons.Count != 0) //make sure if connection list is not empth!
{
//con_list is sorted.
AHAddress last = this_addr;
//the first element of cons is the nearest.
for (int i = 0; i < cons.Count; i++) {
Connection next_c = (Connection)cons[i];
AHAddress next_addr = (AHAddress)next_c.Address;
ISender sender = (ISender) next_c.Edge;
string front = last.ToString();
string back = next_addr.ToString();
string rg_start = start.ToString();
string rg_end = end.ToString();
ArrayList gen_arg = new ArrayList();
if (i==cons.Count -1) { // The last bit
if (left) {
// the left farthest neighbor
gen_arg.Add(front);
gen_arg.Add(rg_end);
}
else {
// the right farthest neighbor
gen_arg.Add(rg_start);
gen_arg.Add(front);
}
}
else {
if (left) { //left connections
gen_arg.Add(front);
gen_arg.Add(back);
}
else { //right connections
gen_arg.Add(back);
gen_arg.Add(front);
}
}
MapReduceInfo mr_info = new MapReduceInfo( (ISender) sender,
new MapReduceArgs(this.TaskName,
mr_args.MapArg,
gen_arg,
mr_args.ReduceArg,
timeout // timeout
));
Log("{0}: {1}, adding address: {2} to sender list, range start: {3}, range end: {4}",
this.TaskName, _node.Address, next_c.Address,
gen_arg[0], gen_arg[1]);
last = next_addr;
retval.Add(mr_info);
}
}
return retval;
}
/**
* Generates tree for bounded broadcast. Algorithm works as follows:
* The goal is to broadcast to all nodes in range [local_address, end).
* Given a range [local_address, b), determine all connections that belong to this range.
* Let the connections be b_1, b_2, ..... b_n.
* To connection bi assign the range [b_i, b_{i+1}).
* To the connection bn assign range [b_n, end).]
*/
public override void GenerateTree(Channel q, MapReduceArgs mr_args)
{
object gen_arg = mr_args.GenArg;
string end_range = gen_arg as string;
Log("generating child tree, range end: {0}.", end_range);
AHAddress end_addr = (AHAddress) AddressParser.Parse(end_range);
AHAddress start_addr = _node.Address as AHAddress;
//we are at the start node, here we go:
ConnectionTable tab = _node.ConnectionTable;
ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
ArrayList retval = new ArrayList();
if (structs.Count > 0) {
Connection curr_con = structs.GetLeftNeighborOf(_node.Address);
int curr_idx = structs.IndexOf(curr_con.Address);
//keep going until we leave the range
int count = 0;
ArrayList con_list = new ArrayList();
while (count++ < structs.Count && ((AHAddress) curr_con.Address).IsBetweenFromLeft(start_addr, end_addr)) {
con_list.Add(curr_con);
//Log("adding connection: {0} to list.", curr_con.Address);
curr_idx = (curr_idx + 1)%structs.Count;
curr_con = structs[curr_idx];
}
Log("{0}: {1}, number of child connections: {2}",
this.TaskName, _node.Address, con_list.Count);
for (int i = 0; i < con_list.Count; i++) {
MapReduceInfo mr_info = null;
ISender sender = null;
Connection con = (Connection) con_list[i];
sender = (ISender) con.Edge;
//check if last connection
if (i == con_list.Count - 1) {
mr_info = new MapReduceInfo( (ISender) sender,
new MapReduceArgs(this.TaskName,
mr_args.MapArg, //map argument
end_range, //generate argument
mr_args.ReduceArg //reduce argument
));
Log("{0}: {1}, adding address: {2} to sender list, range end: {3}",
this.TaskName, _node.Address,
con.Address, end_range);
retval.Add(mr_info);
}
else {
string child_end = ((Connection) con_list[i+1]).Address.ToString();
mr_info = new MapReduceInfo( sender,
new MapReduceArgs(this.TaskName,
mr_args.MapArg,
child_end,
mr_args.ReduceArg));
Log("{0}: {1}, adding address: {2} to sender list, range end: {3}",
this.TaskName, _node.Address,
con.Address, child_end);
retval.Add(mr_info);
}
}
}
q.Enqueue( retval.ToArray(typeof(MapReduceInfo)));
}
