/** Greedy routing. gen_arg is the Address of the destination */ public override void GenerateTree(Channel q, MapReduceArgs mr_args) { object gen_arg = mr_args.GenArg; Log("{0}: {1}, greedy generator called, arg: {2}.", this.TaskName, _node.Address, gen_arg); string address = gen_arg as string; AHAddress a = (AHAddress)AddressParser.Parse(address); ArrayList retval = new ArrayList(); ConnectionTable tab = _node.ConnectionTable; ConnectionList structs = tab.GetConnections(ConnectionType.Structured); Connection next_closest = structs.GetNearestTo((AHAddress)_node.Address, a); if (next_closest != null) { //arguments do not change at all MapReduceInfo mr_info = new MapReduceInfo(next_closest.Edge, mr_args); retval.Add(mr_info); } Log("{0}: {1}, greedy generator returning: {2} senders.", this.TaskName, _node.Address, retval.Count); //Send the result: q.Enqueue(retval.ToArray(typeof(MapReduceInfo))); }
/** * When a node is out of the range, this method is called. * This method tries to find the nearest node to the middle of range using greedty algorithm. * return list of MapReduceInfo */ private List <MapReduceInfo> GenerateTreeOutRange(AHAddress start, AHAddress end, MapReduceArgs mr_args) { List <MapReduceInfo> retval = new List <MapReduceInfo>(); BigInteger up = start.ToBigInteger(); BigInteger down = end.ToBigInteger(); BigInteger mid_range = (up + down) / 2; if (mid_range % 2 == 1) { mid_range = mid_range - 1; } AHAddress mid_addr = new AHAddress(mid_range); //if (!mid_addr.IsBetweenFromLeft(start, end) ) { if (!InRange(mid_addr, start, end)) { mid_range += Address.Half; mid_addr = new AHAddress(mid_range); } ArrayList gen_arg = new ArrayList(); if (NextGreedyClosest(mid_addr) != null) { AHGreedySender ags = new AHGreedySender(_node, mid_addr); string start_range = start.ToString(); string end_range = end.ToString(); gen_arg.Add(start_range); gen_arg.Add(end_range); MapReduceInfo mr_info = new MapReduceInfo((ISender)ags, new MapReduceArgs(this.TaskName, mr_args.MapArg, gen_arg, mr_args.ReduceArg)); Log("{0}: {1}, out of range, moving to the closest node to mid_range: {2} to target node, range start: {3}, range end: {4}", this.TaskName, _node.Address, mid_addr, start, end); retval.Add(mr_info); } else { // cannot find a node in the range. } return(retval); }
/** Greedy routing. gen_arg is the Address of the destination */ public override void GenerateTree(Channel q, MapReduceArgs mr_args) { object gen_arg = mr_args.GenArg; Log("{0}: {1}, greedy generator called, arg: {2}.", this.TaskName, _node.Address, gen_arg); string address = gen_arg as string; AHAddress a = (AHAddress) AddressParser.Parse(address); ArrayList retval = new ArrayList(); ConnectionTable tab = _node.ConnectionTable; ConnectionList structs = tab.GetConnections(ConnectionType.Structured); Connection next_closest = structs.GetNearestTo((AHAddress) _node.Address, a); if (next_closest != null) { //arguments do not change at all MapReduceInfo mr_info = new MapReduceInfo(next_closest.State.Edge, mr_args); retval.Add(mr_info); } Log("{0}: {1}, greedy generator returning: {2} senders.", this.TaskName, _node.Address, retval.Count); //Send the result: q.Enqueue(retval.ToArray(typeof(MapReduceInfo))); }
protected void ChildCallback(object cq, EventArgs arg) { RpcResult child_r; Channel child_q = (Channel)cq; MapReduceInfo mri = (MapReduceInfo)child_q.State; if (LogEnabled) { ProtocolLog.Write(ProtocolLog.MapReduce, String.Format("MapReduce: {0}, handling child result from: {1}.", _node.Address, mri.Sender.ToUri())); } if (child_q.Count == 0) { child_r = new RpcResult(mri.Sender, new AdrException(-32000, "Child did not return")); } else { child_r = (RpcResult)child_q.Dequeue(); } Reduce(child_r); }
/** * Generates tree for bounded broadcast. Algorithm works as follows: * The goal is to broadcast to all nodes in range [local_address, end). * Given a range [local_address, b), determine all connections that belong to this range. * Let the connections be b_1, b_2, ..... b_n. * To connection bi assign the range [b_i, b_{i+1}). * To the connection bn assign range [b_n, end).] */ public override void GenerateTree(Channel q, MapReduceArgs mr_args) { object gen_arg = mr_args.GenArg; string end_range = gen_arg as string; Log("generating child tree, range end: {0}.", end_range); AHAddress end_addr = (AHAddress) AddressParser.Parse(end_range); AHAddress start_addr = _node.Address as AHAddress; //we are at the start node, here we go: ConnectionTable tab = _node.ConnectionTable; ConnectionList structs = tab.GetConnections(ConnectionType.Structured); ArrayList retval = new ArrayList(); if (structs.Count > 0) { Connection curr_con = structs.GetLeftNeighborOf(_node.Address); int curr_idx = structs.IndexOf(curr_con.Address); //keep going until we leave the range int count = 0; List<Connection> con_list = new List<Connection>(); //ArrayList con_list = new ArrayList(); while (count++ < structs.Count && ((AHAddress) curr_con.Address).IsBetweenFromLeft(start_addr, end_addr)) { con_list.Add(curr_con); //Log("adding connection: {0} to list.", curr_con.Address); curr_idx = (curr_idx + 1)%structs.Count; curr_con = structs[curr_idx]; } Log("{0}: {1}, number of child connections: {2}", this.TaskName, _node.Address, con_list.Count); for (int i = 0; i < con_list.Count; i++) { MapReduceInfo mr_info = null; ISender sender = null; Connection con = (Connection) con_list[i]; sender = con.State.Edge; //check if last connection if (i == con_list.Count - 1) { mr_info = new MapReduceInfo( (ISender) sender, new MapReduceArgs(this.TaskName, mr_args.MapArg, //map argument end_range, //generate argument mr_args.ReduceArg //reduce argument )); Log("{0}: {1}, adding address: {2} to sender list, range end: {3}", this.TaskName, _node.Address, con.Address, end_range); retval.Add(mr_info); } else { string child_end = ((Connection) con_list[i+1]).Address.ToString(); mr_info = new MapReduceInfo( sender, new MapReduceArgs(this.TaskName, mr_args.MapArg, child_end, mr_args.ReduceArg)); Log("{0}: {1}, adding address: {2} to sender list, range end: {3}", this.TaskName, _node.Address, con.Address, child_end); retval.Add(mr_info); } } } q.Enqueue( retval.ToArray(typeof(MapReduceInfo))); }
/** * When a node is out of the range, this method is called. * This method tries to find the nearest node to the middle of range using greedty algorithm. * return list of MapReduceInfo */ private List<MapReduceInfo> GenerateTreeOutRange(AHAddress start, AHAddress end, MapReduceArgs mr_args) { List<MapReduceInfo> retval = new List<MapReduceInfo>(); BigInteger up = start.ToBigInteger(); BigInteger down = end.ToBigInteger(); BigInteger mid_range = (up + down) /2; if (mid_range % 2 == 1) {mid_range = mid_range -1; } AHAddress mid_addr = new AHAddress(mid_range); //if (!mid_addr.IsBetweenFromLeft(start, end) ) { if (!InRange(mid_addr, start, end) ) { mid_range += Address.Half; mid_addr = new AHAddress(mid_range); } ArrayList gen_arg = new ArrayList(); if (NextGreedyClosest(mid_addr) != null ) { AHGreedySender ags = new AHGreedySender(_node, mid_addr); string start_range = start.ToString(); string end_range = end.ToString(); gen_arg.Add(start_range); gen_arg.Add(end_range); MapReduceInfo mr_info = new MapReduceInfo( (ISender) ags, new MapReduceArgs(this.TaskName, mr_args.MapArg, gen_arg, mr_args.ReduceArg)); Log("{0}: {1}, out of range, moving to the closest node to mid_range: {2} to target node, range start: {3}, range end: {4}", this.TaskName, _node.Address, mid_addr, start, end); retval.Add(mr_info); } else { // cannot find a node in the range. } return retval; }
/** * Generate tree within the range. * return list of MapReduceInfo */ private List<MapReduceInfo> GenerateTreeInRange(AHAddress start, AHAddress end, List<Connection> cons, bool left, MapReduceArgs mr_args) { //Divide the range and trigger bounded broadcasting again in divided range starting with neighbor. //Deivided ranges are (start, n_1), (n_1, n_2), ... , (n_m, end) AHAddress this_minus2 = new AHAddress(_this_addr.ToBigInteger()-2); AHAddress this_plus2 = new AHAddress(_this_addr.ToBigInteger()+2); List<MapReduceInfo> retval = new List<MapReduceInfo>(); if (cons.Count != 0) //make sure if connection list is not empty! { //con_list is sorted. AHAddress last; if (left) { last = end; } else { last = start; } string rg_start, rg_end; //the first element of cons is the nearest. //Let's start with the farthest neighbor first. for (int i = (cons.Count-1); i >= 0; i--) { ArrayList gen_arg = new ArrayList(); Connection next_c = cons[i]; AHAddress next_addr = (AHAddress)next_c.Address; ISender sender = next_c.State.Edge; if (i==0) { // The last bit if (left) { // the left nearest neighbor rg_start = this_plus2.ToString(); rg_end = last.ToString(); } else { // the right nearest neighbor rg_start = last.ToString(); rg_end = this_minus2.ToString(); } } else { if (left) { //left connections rg_start = next_addr.ToString(); rg_end = last.ToString(); } else { //right connections rg_start = last.ToString(); rg_end = next_addr.ToString(); } } gen_arg.Add(rg_start); gen_arg.Add(rg_end); MapReduceInfo mr_info = new MapReduceInfo( sender, new MapReduceArgs(this.TaskName, mr_args.MapArg, gen_arg, mr_args.ReduceArg)); Log("{0}: {1}, adding address: {2} to sender list, range start: {3}, range end: {4}", this.TaskName, _node.Address, next_c.Address, gen_arg[0], gen_arg[1]); if (left) { last = new AHAddress(next_addr.ToBigInteger()-2); } else { last = new AHAddress(next_addr.ToBigInteger()+2); } retval.Add(mr_info); } } return retval; }
/** * Generates tree for bounded broadcast. Algorithm works as follows: * The goal is to broadcast to all nodes in range [local_address, end). * Given a range [local_address, b), determine all connections that belong to this range. * Let the connections be b_1, b_2, ..... b_n. * To connection bi assign the range [b_i, b_{i+1}). * To the connection bn assign range [b_n, end).] */ public override void GenerateTree(Channel q, MapReduceArgs mr_args) { object gen_arg = mr_args.GenArg; string end_range = gen_arg as string; Log("generating child tree, range end: {0}.", end_range); AHAddress end_addr = (AHAddress)AddressParser.Parse(end_range); AHAddress start_addr = _node.Address as AHAddress; //we are at the start node, here we go: ConnectionTable tab = _node.ConnectionTable; ConnectionList structs = tab.GetConnections(ConnectionType.Structured); ArrayList retval = new ArrayList(); if (structs.Count > 0) { Connection curr_con = structs.GetLeftNeighborOf(_node.Address); int curr_idx = structs.IndexOf(curr_con.Address); //keep going until we leave the range int count = 0; ArrayList con_list = new ArrayList(); while (count++ < structs.Count && ((AHAddress)curr_con.Address).IsBetweenFromLeft(start_addr, end_addr)) { con_list.Add(curr_con); //Log("adding connection: {0} to list.", curr_con.Address); curr_idx = (curr_idx + 1) % structs.Count; curr_con = structs[curr_idx]; } Log("{0}: {1}, number of child connections: {2}", this.TaskName, _node.Address, con_list.Count); for (int i = 0; i < con_list.Count; i++) { MapReduceInfo mr_info = null; ISender sender = null; Connection con = (Connection)con_list[i]; sender = (ISender)con.Edge; //check if last connection if (i == con_list.Count - 1) { mr_info = new MapReduceInfo((ISender)sender, new MapReduceArgs(this.TaskName, mr_args.MapArg, //map argument end_range, //generate argument mr_args.ReduceArg //reduce argument )); Log("{0}: {1}, adding address: {2} to sender list, range end: {3}", this.TaskName, _node.Address, con.Address, end_range); retval.Add(mr_info); } else { string child_end = ((Connection)con_list[i + 1]).Address.ToString(); mr_info = new MapReduceInfo(sender, new MapReduceArgs(this.TaskName, mr_args.MapArg, child_end, mr_args.ReduceArg)); Log("{0}: {1}, adding address: {2} to sender list, range end: {3}", this.TaskName, _node.Address, con.Address, child_end); retval.Add(mr_info); } } } q.Enqueue(retval.ToArray(typeof(MapReduceInfo))); }
/** * Generate tree within the range. * return list of MapReduceInfo */ private List <MapReduceInfo> GenerateTreeInRange(AHAddress start, AHAddress end, List <Connection> cons, bool left, MapReduceArgs mr_args) { //Divide the range and trigger bounded broadcasting again in divided range starting with neighbor. //Deivided ranges are (start, n_1), (n_1, n_2), ... , (n_m, end) AHAddress this_minus2 = new AHAddress(_this_addr.ToBigInteger() - 2); AHAddress this_plus2 = new AHAddress(_this_addr.ToBigInteger() + 2); List <MapReduceInfo> retval = new List <MapReduceInfo>(); if (cons.Count != 0) //make sure if connection list is not empty! { //con_list is sorted. AHAddress last; if (left) { last = end; } else { last = start; } string rg_start, rg_end; //the first element of cons is the nearest. //Let's start with the farthest neighbor first. for (int i = (cons.Count - 1); i >= 0; i--) { ArrayList gen_arg = new ArrayList(); Connection next_c = cons[i]; AHAddress next_addr = (AHAddress)next_c.Address; ISender sender = next_c.State.Edge; if (i == 0) // The last bit { if (left) { // the left nearest neighbor rg_start = this_plus2.ToString(); rg_end = last.ToString(); } else { // the right nearest neighbor rg_start = last.ToString(); rg_end = this_minus2.ToString(); } } else { if (left) //left connections { rg_start = next_addr.ToString(); rg_end = last.ToString(); } else //right connections { rg_start = last.ToString(); rg_end = next_addr.ToString(); } } gen_arg.Add(rg_start); gen_arg.Add(rg_end); MapReduceInfo mr_info = new MapReduceInfo(sender, new MapReduceArgs(this.TaskName, mr_args.MapArg, gen_arg, mr_args.ReduceArg)); Log("{0}: {1}, adding address: {2} to sender list, range start: {3}, range end: {4}", this.TaskName, _node.Address, next_c.Address, gen_arg[0], gen_arg[1]); if (left) { last = new AHAddress(next_addr.ToBigInteger() - 2); } else { last = new AHAddress(next_addr.ToBigInteger() + 2); } retval.Add(mr_info); } } return(retval); }
/* * This cannot change our Reducing state. */ public State UpdateTree(MapReduceInfo[] tree) { if( Tree != null ) { throw new Exception("MapReduce: Tree already set"); } bool done = Done; if( ReduceResult != DEFAULT_OBJ ) { /* * We have already reduced the map result, we are done, ONLY if * the tree is empty: */ done = done || (tree.Length == 0); } else { //We haven't yet reduced the map result //Done should be false } return new State(MapResult, tree, ChildReductions, ReduceResult, done, Pending, Reducing); }
protected State(object map, MapReduceInfo[] tree, int cred, object reduce, bool done, ImmutableList<RpcResult> pend, bool reducing) { MapResult = map; Tree = tree; ChildReductions = cred; ReduceResult = reduce; Done = done; Pending = pend; Reducing = reducing; }