public override MapReduceInfo[] GenerateTree(MapReduceArgs mr_args) {
object gen_arg = mr_args.GenArg;
Log("{0}: {1}, greedy generator called, arg: {2}.",
this.TaskName, _node.Address, gen_arg);
string address = gen_arg as string;
AHAddress a = (AHAddress) AddressParser.Parse(address);
ArrayList retval = new ArrayList();
ConnectionTable tab = _node.ConnectionTable;
ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
Connection next_closest = structs.GetNearestTo((AHAddress) _node.Address, a);
if (next_closest != null) {
int timeout = -1;
if (mr_args.MillisecTimeout >= 0) { //only if it is a valid timeout
timeout = (int) (TIMEOUT_FRACTION*mr_args.MillisecTimeout);
}
MapReduceInfo mr_info = new MapReduceInfo( (ISender) next_closest.Edge,
new MapReduceArgs(this.TaskName,//arguments do not change much
mr_args.MapArg,
mr_args.GenArg,
mr_args.ReduceArg,
timeout));//only timeout changes
retval.Add(mr_info);
}
Log("{0}: {1}, greedy generator returning: {2} senders.",
this.TaskName, _node.Address, retval.Count);
return (MapReduceInfo[]) retval.ToArray(typeof(MapReduceInfo));
}
/**
* Generates tree for bounded broadcast. Algorithm works as follows:
* The goal is to broadcast to all nodes in range (start, end).
* Given a range (a, b), determine all connections that belong to this range.
* Let the left connections be l_1, l_2, ..... l_n.
* Let the right connections be r_1, r_2, ... , r_n.
* To left connection l_i assign the range [b_{i-1}, b_i).
* To right connection r_i assign the range [r_i, r_{i-1}]
* To the connection ln assign range [l_{n-1}, end)
* To the connection rn assign range (start, r_{n-1}]
*/
public override MapReduceInfo[] GenerateTree(MapReduceArgs mr_args)
{
ArrayList gen_list = mr_args.GenArg as ArrayList;
string start_range = gen_list[0] as string;
string end_range = gen_list[1] as string;
AHAddress this_addr = _node.Address as AHAddress;
AHAddress start_addr = (AHAddress) AddressParser.Parse(start_range);
AHAddress end_addr = (AHAddress) AddressParser.Parse(end_range);
Log("generating child tree, range start: {0}, range end: {1}.", start_range, end_range);
//we are at the start node, here we go:
ConnectionTable tab = _node.ConnectionTable;
ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
ArrayList retval = new ArrayList();
int timeout = -1;
if (mr_args.MillisecTimeout >= 0) { //only if it is a valid timeout
timeout = (int) (TIMEOUT_FRACTION*mr_args.MillisecTimeout);
}
if (InRange(this_addr, start_addr, end_addr)) {
if (structs.Count > 0) {
//make connection list in the range.
//left connection list is a list of neighbors which are in the range (this node, end of range)
//right connection list is a list of neighbors which are in the range (start of range, this node)
ArrayList cons = GetConnectionInfo(this_addr, start_addr, end_addr, structs);
List<Connection> left_cons = cons[0] as List<Connection>;
List<Connection> right_cons = cons[1] as List<Connection>;
retval = GenerateTreeInRange(this_addr, start_addr, end_addr, left_cons, true, mr_args, timeout);
ArrayList ret_right = GenerateTreeInRange(this_addr, start_addr, end_addr, right_cons, false, mr_args, timeout);
retval.AddRange(ret_right);
}
else { //this node is a leaf node.
MapReduceInfo mr_info = null;
retval.Add(mr_info);
//Console.WriteLine("no connection in the range: return null info");
}
//Console.WriteLine("````````````retval.Count: {0}",retval.Count);
}
else { // _node is out of range. Just pass it to the closest to the middle of range.
retval = GenerateTreeOutRange(start_addr, end_addr, mr_args, timeout);
}
return (MapReduceInfo[]) retval.ToArray(typeof(MapReduceInfo));
}
public override MapReduceInfo[] GenerateTree(MapReduceArgs mr_args) {
object gen_arg = mr_args.GenArg;
Log("{0}: {1}, greedy generator called, arg: {2}.",
this.TaskName, _node.Address, gen_arg);
string address = gen_arg as string;
AHAddress a = (AHAddress) AddressParser.Parse(address);
ArrayList retval = new ArrayList();
ConnectionTable tab = _node.ConnectionTable;
ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
Connection next_closest = structs.GetNearestTo((AHAddress) _node.Address, a);
if (next_closest != null) {
MapReduceInfo mr_info = new MapReduceInfo( (ISender) next_closest.Edge,
mr_args); //arguments do not change at all
retval.Add(mr_info);
}
Log("{0}: {1}, greedy generator returning: {2} senders.",
this.TaskName, _node.Address, retval.Count);
return (MapReduceInfo[]) retval.ToArray(typeof(MapReduceInfo));
}
/**
* This dispatches the particular methods this class provides.
* Currently, the only invokable method is:
* "Start".
*/
public void HandleRpc(ISender caller, string method, IList args, object req_state) {
int part_idx = method.IndexOf(':');
if( part_idx == -1 ) {
if (method == "Start") {
IDictionary ht = (IDictionary) args[0];
MapReduceArgs mr_args = new MapReduceArgs(ht);
string task_name = mr_args.TaskName;
MapReduceTask task;
if (_name_to_task.TryGetValue(task_name, out task)) {
MapReduceComputation mr = new MapReduceComputation(_node, req_state, task, mr_args);
mr.Start();
}
else {
throw new AdrException(-32608, "No mapreduce task with name: " + task_name);
}
}
else if( method == "AddHandler" ) {
//Make sure this is local:
ISender tmp_call = caller;
bool islocal = tmp_call is Node;
while(!islocal && tmp_call is IWrappingSender) {
tmp_call = ((IWrappingSender)tmp_call).WrappedSender;
islocal = tmp_call is Node;
}
if( !islocal ) {
throw new AdrException(-32601, "AddHandler only valid for local callers");
}
SubscribeTask(new RpcMapReduceTask(_node, (IDictionary)args[0]));
_rpc.SendResult(req_state, null);
}
else {
throw new AdrException(-32601, "No Handler for method: " + method);
}
}
else {
//This is a reference to a specific part of a task:
string part = method.Substring(0, part_idx);
string task_name = method.Substring(part_idx + 1);
MapReduceTask task;
if(false == _name_to_task.TryGetValue(task_name, out task)) {
throw new AdrException(-32608, "No mapreduce task with name: " + task_name);
}
if( part == "tree" ) {
var mra = new MapReduceArgs((IDictionary)args[0]);
var tree_res = new Channel(1, req_state);
tree_res.CloseEvent += this.HandleTree;
task.GenerateTree(tree_res, mra);
}
else if( part == "reduce" ) {
//Prepare the RpcResult:
var rres_d = (IDictionary)args[2];
ISender send = SenderFactory.CreateInstance(_node, (string)rres_d["sender"]);
var rres = new RpcResult(send, rres_d["result"]);
Channel reduce_res = new Channel(1, req_state);
reduce_res.CloseEvent += this.HandleReduce;
task.Reduce(reduce_res, args[0], args[1], rres);
}
else if( part == "map" ) {
Channel map_res = new Channel(1, req_state);
map_res.CloseEvent += this.HandleMap;
task.Map(map_res, args[0]);
}
else {
throw new AdrException(-32608,
String.Format("No mapreduce task({0}) part with name: {1}", task_name, part));
}
}
}
protected void TreeHandler(object o, EventArgs eargs) {
Channel result = (Channel)o;
Channel q = (Channel)result.State;
try {
RpcResult r = (RpcResult)result.Dequeue();
var mris = new List<MapReduceInfo>();
foreach(IDictionary d in (IList)r.Result) {
var uri = (string)d["sender"];
var sender = SenderFactory.CreateInstance(_node, uri);
var args = new MapReduceArgs((IDictionary)d["args"]);
mris.Add(new MapReduceInfo(sender, args));
}
q.Enqueue(mris.ToArray());
}
catch(Exception x) {
//Some kind of problem:
q.Enqueue(x);
}
}
public override void GenerateTree(Channel q, MapReduceArgs args) {
Channel result = new Channel(1, q);
result.CloseEvent += this.TreeHandler;
_node.Rpc.Invoke(_tree.First, result, _tree.Second, args.ToHashtable());
}
/** tree generator function.
* @param q The channel into which to put exactly one IList<MapReduceInfo>
* @param args the MapReduceArgs for this call
*/
public virtual void GenerateTree(Channel q, MapReduceArgs args) {
throw new NotImplementedException();
}
/** tree generator function. */ public abstract MapReduceInfo[] GenerateTree(MapReduceArgs args);
/**
* Constructor
* @param node local node
* @param state RPC related state.
* @param task map-reduce task.
* @param args arguments to the map reduce task.
*/
public MapReduceComputation(Node node, object state,
MapReduceTask task,
MapReduceArgs args)
{
_node = node;
_rpc = RpcManager.GetInstance(node);
_mr_request_state = state;
_mr_task = task;
_mr_args = args;
_queue_to_child = new Hashtable();
_sync = new object();
_finished = false;
}
/**
* This dispatches the particular methods this class provides.
* Currently, the only invokable method is:
* "Start".
*/
public void HandleRpc(ISender caller, string method, IList args, object req_state) {
if (method == "Start") {
Hashtable ht = (Hashtable) args[0];
MapReduceArgs mr_args = new MapReduceArgs(ht);
string task_name = mr_args.TaskName;
MapReduceTask task = null;
lock(_sync) {
task = (MapReduceTask) _name_to_task[task_name];
}
if (task != null) {
Start(task, mr_args, req_state);
}
else {
throw new AdrException(-32608, "No mapreduce task with name: " + task_name);
}
}
else {
throw new AdrException(-32601, "No Handler for method: " + method);
}
}
/**
* When a node is out of the range, this method is called.
* This method tries to find the nearest node to the middle of range using greedty algorithm.
* return list of MapReduceInfo
*/
private ArrayList GenerateTreeOutRange(AHAddress start, AHAddress end, MapReduceArgs mr_args, int timeout) {
ArrayList retval = new ArrayList();
BigInteger up = start.ToBigInteger();
BigInteger down = end.ToBigInteger();
BigInteger mid_range = (up + down) /2;
if (mid_range % 2 == 1) {mid_range = mid_range -1; }
AHAddress mid_addr = new AHAddress(mid_range);
if (!mid_addr.IsBetweenFromLeft(start, end) ) {
mid_range += Address.Half;
mid_addr = new AHAddress(mid_range);
}
ArrayList gen_arg = new ArrayList();
if (NextGreedyClosest(mid_addr) != null ) {
AHGreedySender ags = new AHGreedySender(_node, mid_addr);
string start_range = start.ToString();
string end_range = end.ToString();
gen_arg.Add(start_range);
gen_arg.Add(end_range);
MapReduceInfo mr_info = new MapReduceInfo( (ISender) ags,
new MapReduceArgs(this.TaskName,
mr_args.MapArg,
gen_arg,
mr_args.ReduceArg,
timeout));
Log("{0}: {1}, out of range, moving to the closest node to mid_range: {2} to target node, range start: {3}, range end: {4}",
this.TaskName, _node.Address, mid_addr, start, end);
retval.Add(mr_info);
}
else {
// cannot find a node in the range.
}
return retval;
}
/**
* Generate tree within the range.
* return list of MapReduceInfo
*/
private ArrayList GenerateTreeInRange(AHAddress this_addr, AHAddress start, AHAddress end, List<Connection> cons, bool left, MapReduceArgs mr_args, int timeout) {
//Divide the range and trigger bounded broadcasting again in divided range starting with neighbor.
//Deivided ranges are (start, n_1), (n_1, n_2), ... , (n_m, end)
ArrayList retval = new ArrayList();
if (cons.Count != 0) //make sure if connection list is not empth!
{
//con_list is sorted.
AHAddress last = this_addr;
//the first element of cons is the nearest.
for (int i = 0; i < cons.Count; i++) {
Connection next_c = (Connection)cons[i];
AHAddress next_addr = (AHAddress)next_c.Address;
ISender sender = (ISender) next_c.Edge;
string front = last.ToString();
string back = next_addr.ToString();
string rg_start = start.ToString();
string rg_end = end.ToString();
ArrayList gen_arg = new ArrayList();
if (i==cons.Count -1) { // The last bit
if (left) {
// the left farthest neighbor
gen_arg.Add(front);
gen_arg.Add(rg_end);
}
else {
// the right farthest neighbor
gen_arg.Add(rg_start);
gen_arg.Add(front);
}
}
else {
if (left) { //left connections
gen_arg.Add(front);
gen_arg.Add(back);
}
else { //right connections
gen_arg.Add(back);
gen_arg.Add(front);
}
}
MapReduceInfo mr_info = new MapReduceInfo( (ISender) sender,
new MapReduceArgs(this.TaskName,
mr_args.MapArg,
gen_arg,
mr_args.ReduceArg,
timeout // timeout
));
Log("{0}: {1}, adding address: {2} to sender list, range start: {3}, range end: {4}",
this.TaskName, _node.Address, next_c.Address,
gen_arg[0], gen_arg[1]);
last = next_addr;
retval.Add(mr_info);
}
}
return retval;
}
/**
* Generates tree for bounded broadcast. Algorithm works as follows:
* The goal is to broadcast to all nodes in range [local_address, end).
* Given a range [local_address, b), determine all connections that belong to this range.
* Let the connections be b_1, b_2, ..... b_n.
* To connection bi assign the range [b_i, b_{i+1}).
* To the connection bn assign range [b_n, end).]
*/
public override void GenerateTree(Channel q, MapReduceArgs mr_args)
{
object gen_arg = mr_args.GenArg;
string end_range = gen_arg as string;
Log("generating child tree, range end: {0}.", end_range);
AHAddress end_addr = (AHAddress) AddressParser.Parse(end_range);
AHAddress start_addr = _node.Address as AHAddress;
//we are at the start node, here we go:
ConnectionTable tab = _node.ConnectionTable;
ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
ArrayList retval = new ArrayList();
if (structs.Count > 0) {
Connection curr_con = structs.GetLeftNeighborOf(_node.Address);
int curr_idx = structs.IndexOf(curr_con.Address);
//keep going until we leave the range
int count = 0;
ArrayList con_list = new ArrayList();
while (count++ < structs.Count && ((AHAddress) curr_con.Address).IsBetweenFromLeft(start_addr, end_addr)) {
con_list.Add(curr_con);
//Log("adding connection: {0} to list.", curr_con.Address);
curr_idx = (curr_idx + 1)%structs.Count;
curr_con = structs[curr_idx];
}
Log("{0}: {1}, number of child connections: {2}",
this.TaskName, _node.Address, con_list.Count);
for (int i = 0; i < con_list.Count; i++) {
MapReduceInfo mr_info = null;
ISender sender = null;
Connection con = (Connection) con_list[i];
sender = (ISender) con.Edge;
//check if last connection
if (i == con_list.Count - 1) {
mr_info = new MapReduceInfo( (ISender) sender,
new MapReduceArgs(this.TaskName,
mr_args.MapArg, //map argument
end_range, //generate argument
mr_args.ReduceArg //reduce argument
));
Log("{0}: {1}, adding address: {2} to sender list, range end: {3}",
this.TaskName, _node.Address,
con.Address, end_range);
retval.Add(mr_info);
}
else {
string child_end = ((Connection) con_list[i+1]).Address.ToString();
mr_info = new MapReduceInfo( sender,
new MapReduceArgs(this.TaskName,
mr_args.MapArg,
child_end,
mr_args.ReduceArg));
Log("{0}: {1}, adding address: {2} to sender list, range end: {3}",
this.TaskName, _node.Address,
con.Address, child_end);
retval.Add(mr_info);
}
}
}
q.Enqueue( retval.ToArray(typeof(MapReduceInfo)));
}
public MapReduceInfo(ISender sender, MapReduceArgs args) {
Sender = sender;
Args = args;
}
/**
* Starts a map-reduce computation.
* @param task map reduce task to start.
* @param args arguments for the map-reduce task.
* @param req_state RPC related state for the invocation.
*/
protected void Start(MapReduceTask task, MapReduceArgs args, object req_state) {
MapReduceComputation mr = new MapReduceComputation(_node, req_state, task, args);
mr.Start();
}
/**
* Constructor
* @param node local node
* @param state RPC related state.
* @param task map-reduce task.
* @param args arguments to the map reduce task.
*/
public MapReduceComputation(Node node, object state,
MapReduceTask task,
MapReduceArgs args)
{
_node = node;
_rpc = node.Rpc;
_mr_request_state = state;
_mr_task = task;
_mr_args = args;
//Here is our state variable:
_state = new State();
_result = State.DEFAULT_OBJ;
}
/**
* Constructor
* @param node local node
* @param state RPC related state.
* @param task map-reduce task.
* @param args arguments to the map reduce task.
*/
public MapReduceComputation(Node node, object state,
MapReduceTask task,
MapReduceArgs args)
{
_node = node;
_rpc = RpcManager.GetInstance(node);
_mr_request_state = state;
_mr_task = task;
_mr_args = args;
_queue_to_child = new Hashtable();
_sync = new object();
_finished = false;
_start_time = DateTime.UtcNow;
//register with the node hearbeat to for computation to finish.
lock(_sync) {
_node.HeartBeatEvent += new EventHandler(this.CheckTimeout);
}
}
