public override object Reduce(object reduce_arg, object current_result, RpcResult child_rpc, out bool done) { done = false; ISender child_sender = child_rpc.ResultSender; //the following can throw an exception, will be handled by the framework object child_result = child_rpc.Result; //child result is a valid result if (current_result == null) { return child_result; } ArrayList retval = current_result as ArrayList; IDictionary my_entry = (IDictionary) retval[0]; Edge e = child_sender as Edge; my_entry["next_con"] = e.ToString(); retval.AddRange((IList) child_result); if (LogEnabled) { ProtocolLog.Write(ProtocolLog.MapReduce, String.Format("{0}: {1}, reduce list count: {2}.", this.TaskName, _node.Address, retval.Count)); } return retval; }
public override void Reduce(Channel q, object reduce_arg, object current_result, RpcResult child_rpc) { bool done = false; ISender child_sender = child_rpc.ResultSender; //the following can throw an exception, will be handled by the framework object child_result = child_rpc.Result; //child result is a valid result if (current_result == null) { q.Enqueue(new Brunet.Util.Pair<object, bool>(child_result, done)); return; } ArrayList retval = current_result as ArrayList; IDictionary my_entry = (IDictionary) retval[0]; my_entry["next_con"] = child_sender.ToUri(); retval.AddRange((IList) child_result); if (LogEnabled) { ProtocolLog.Write(ProtocolLog.MapReduce, String.Format("{0}: {1}, reduce list count: {2}.", this.TaskName, _node.Address, retval.Count)); } q.Enqueue(new Brunet.Util.Pair<object, bool>(retval, done)); }
public override object Reduce(object reduce_arg, object current_result, RpcResult child_rpc, out bool done) { done = false; //ISender child_sender = child_rpc.ResultSender; //the following can throw an exception, will be handled by the framework object child_result = child_rpc.Result; //child result is a valid result if (current_result == null) { return child_result; } IDictionary my_entry = current_result as IDictionary; IDictionary value = child_result as IDictionary; int max_height = (int) my_entry["height"]; int count = (int) my_entry["count"]; int y = (int) value["count"]; my_entry["count"] = count + y; int z = (int) value["height"] + 1; if (z > max_height) { my_entry["height"] = z; } return my_entry; }
public override void Reduce(Channel q, object reduce_arg, object current_result, RpcResult child_rpc) { bool done = false; //ISender child_sender = child_rpc.ResultSender; //the following can throw an exception, will be handled by the framework object child_result = child_rpc.Result; //child result is a valid result if (current_result == null) { q.Enqueue(new Brunet.Util.Pair<object, bool>(child_result, done)); return; } IDictionary my_entry = current_result as IDictionary; IDictionary value = child_result as IDictionary; int max_height = (int) my_entry["height"]; int count = (int) my_entry["count"]; int y = (int) value["count"]; my_entry["count"] = count + y; int z = (int) value["height"] + 1; if (z > max_height) { my_entry["height"] = z; } q.Enqueue(new Brunet.Util.Pair<object, bool>(my_entry, done)); }
/** * When an error comes in, this handles it */ public void HandleError(ReqrepManager man, int message_number, ReqrepManager.ReqrepError err, ISender ret_path, object state) { Exception x = null; RpcRequestState rs = (RpcRequestState) state; Channel bq = rs.Results; switch(err) { case ReqrepManager.ReqrepError.NoHandler: x = new AdrException(-32601, "No RPC Handler on remote host"); break; case ReqrepManager.ReqrepError.HandlerFailure: x = new AdrException(-32603, "The remote RPC System had a problem"); break; case ReqrepManager.ReqrepError.Timeout: //In this case we close the Channel: if( bq != null ) { bq.Close(); } break; case ReqrepManager.ReqrepError.Send: if( rs.RpcTarget is Edge ) { Edge e = (Edge)rs.RpcTarget; //This definitely won't get any more responses: if( e.IsClosed && bq != null ) { bq.Close(); } } //We had some problem sending, but ignore it for now break; } if( x != null && (bq != null) ) { RpcResult res = new RpcResult(ret_path, x); bq.Enqueue(res); } }
/** * Implements the IReplyHandler (also provides some light-weight statistics) */ public bool HandleReply(ReqrepManager man, ReqrepManager.ReqrepType rt, int mid, PType prot, MemBlock payload, ISender ret_path, ReqrepManager.Statistics statistics, object state) { RpcRequestState rs = (RpcRequestState) state; //ISender target = rs.RpcTarget; Channel bq = rs.Results; if( bq != null ) { object data = AdrConverter.Deserialize(payload); RpcResult res = new RpcResult(ret_path, data, statistics); bq.Enqueue(res); //Keep listening unless the queue is closed return (!bq.Closed); } else { //If they didn't even pass us a queue, I guess they didn't want to //listen too long return false; } }
protected void ChildCallback(object cq, EventArgs arg) { RpcResult child_r; Channel child_q = (Channel)cq; MapReduceInfo mri = (MapReduceInfo)child_q.State; if (LogEnabled) { ProtocolLog.Write(ProtocolLog.MapReduce, String.Format("MapReduce: {0}, handling child result from: {1}.", _node.Address, mri.Sender.ToUri())); } if( child_q.Count == 0 ) { child_r = new RpcResult(mri.Sender, new AdrException(-32000, "Child did not return")); } else { child_r = (RpcResult)child_q.Dequeue(); } Reduce(child_r); }
protected void Reduce(RpcResult child_r) { //The usual transactional bit: State state = _state; State old_state; State new_state; do { old_state = state; new_state = old_state.AddChildResult(child_r); state = Interlocked.CompareExchange<State>(ref _state, new_state, old_state); } while( state != old_state); //If we need to start a new reduce, it's the latest value: TryNextReduce(new_state, old_state, child_r, false); }
/** * @return true if we successfully started the next reduce */ protected bool TryNextReduce(State new_s, State old_s, RpcResult v, bool cont) { if( new_s.Done ) { SendResult( new_s.ReduceResult ); return false; } bool start_red = new_s.Reducing && (cont || (false == old_s.Reducing)); if (LogEnabled) { ProtocolLog.Write(ProtocolLog.MapReduce, String.Format("MapReduce: {0}, TryNextReduce: {1}.", _node.Address, start_red)); } if( start_red ) { Channel r_chan = new Channel(1, v); r_chan.CloseEvent += this.ReduceHandler; object startval = new_s.ReduceResult == State.DEFAULT_OBJ ? null : new_s.ReduceResult; if (LogEnabled) { ProtocolLog.Write(ProtocolLog.MapReduce, String.Format("MapReduce: {0} abt to Reduce({1},{2},{3})", _node.Address, _mr_args.ReduceArg, startval, v)); } try { _mr_task.Reduce(r_chan, _mr_args.ReduceArg, startval, v); } catch(Exception x) { //Reduce is where we do error handling, if that doesn't work, oh well: if (LogEnabled) { ProtocolLog.Write(ProtocolLog.MapReduce, String.Format("MapReduce: {0}, reduce threw: {1}.", _node.Address, x)); } SendResult(x); return false; } } return start_red; }
/* * We always reduce our map call first, then children. */ public State AddChildResult(RpcResult child) { if( Reducing ) { //We need to add this to pending var pend = new ImmutableList<RpcResult>(child, Pending); return new State(MapResult, Tree, ChildReductions, ReduceResult, Done, pend, true); } else if( MapResult == DEFAULT_OBJ ) { /* * We are not reducing AND we have not finished the first * Reduce, that means we are waiting for the Map result. */ var pend = new ImmutableList<RpcResult>(child, Pending); return new State(MapResult, Tree, ChildReductions, ReduceResult, Done, pend, false); } else { /* * In this case, we are not already reducing, and we have reduced * the MapResult */ return new State(MapResult, Tree, ChildReductions, ReduceResult, Done, ImmutableList<RpcResult>.Empty, true); } }
/** * This dispatches the particular methods this class provides. * Currently, the only invokable method is: * "Start". */ public void HandleRpc(ISender caller, string method, IList args, object req_state) { int part_idx = method.IndexOf(':'); if( part_idx == -1 ) { if (method == "Start") { IDictionary ht = (IDictionary) args[0]; MapReduceArgs mr_args = new MapReduceArgs(ht); string task_name = mr_args.TaskName; MapReduceTask task; if (_name_to_task.TryGetValue(task_name, out task)) { MapReduceComputation mr = new MapReduceComputation(_node, req_state, task, mr_args); mr.Start(); } else { throw new AdrException(-32608, "No mapreduce task with name: " + task_name); } } else if( method == "AddHandler" ) { //Make sure this is local: ISender tmp_call = caller; bool islocal = tmp_call is Node; while(!islocal && tmp_call is IWrappingSender) { tmp_call = ((IWrappingSender)tmp_call).WrappedSender; islocal = tmp_call is Node; } if( !islocal ) { throw new AdrException(-32601, "AddHandler only valid for local callers"); } SubscribeTask(new RpcMapReduceTask(_node, (IDictionary)args[0])); _rpc.SendResult(req_state, null); } else { throw new AdrException(-32601, "No Handler for method: " + method); } } else { //This is a reference to a specific part of a task: string part = method.Substring(0, part_idx); string task_name = method.Substring(part_idx + 1); MapReduceTask task; if(false == _name_to_task.TryGetValue(task_name, out task)) { throw new AdrException(-32608, "No mapreduce task with name: " + task_name); } if( part == "tree" ) { var mra = new MapReduceArgs((IDictionary)args[0]); var tree_res = new Channel(1, req_state); tree_res.CloseEvent += this.HandleTree; task.GenerateTree(tree_res, mra); } else if( part == "reduce" ) { //Prepare the RpcResult: var rres_d = (IDictionary)args[2]; ISender send = SenderFactory.CreateInstance(_node, (string)rres_d["sender"]); var rres = new RpcResult(send, rres_d["result"]); Channel reduce_res = new Channel(1, req_state); reduce_res.CloseEvent += this.HandleReduce; task.Reduce(reduce_res, args[0], args[1], rres); } else if( part == "map" ) { Channel map_res = new Channel(1, req_state); map_res.CloseEvent += this.HandleMap; task.Map(map_res, args[0]); } else { throw new AdrException(-32608, String.Format("No mapreduce task({0}) part with name: {1}", task_name, part)); } } }
public override void Reduce(Channel q, object reduce_arg, object current_result, RpcResult child_rpc) { Channel result = new Channel(1, q); result.CloseEvent += this.ReduceHandler; var childrpc_ht = new Hashtable(); ISender rsend = child_rpc.ResultSender; childrpc_ht["sender"] = rsend != null ? rsend.ToUri() : "sender:localnode"; try { //If this is an exception, this will throw childrpc_ht["result"] = child_rpc.Result; } catch(Exception x) { childrpc_ht["result"] = x; } _node.Rpc.Invoke(_reduce.First, result, _reduce.Second, reduce_arg, current_result, childrpc_ht); }
/** * reduce function. This reduces the local and children results into one. * This is also the error handling function. Any exceptions must be * handled by this method, if not, the computation stops immediately and * sends the exception back up the tree. * * @param q the Channel into which a Brunet.Util.Pair<object, bool> is enqueued, * if the second item is true, we stop querying nodes * @param reduce_arg arguments for the reduce * @param current_result accumulated result of reductions * @param child_rpc result from child computation */ public virtual void Reduce(Channel q, object reduce_arg, object current_result, RpcResult child_rpc) { throw new NotImplementedException(); }
/** * reduce function. * @param reduce_arg arguments for the reduce * @param current_result accumulated result of reductions * @param child_rpc result from child computation * @param done out parameter (is the stopping criteria met) * @param child_results hashtable containing results from each child */ public abstract object Reduce(object reduce_arg, object current_result, RpcResult child_rpc, out bool done);
public override void Reduce(Channel q, object reduce_arg, object current_val, RpcResult child_r) { var rest = child_r.Result as IEnumerable; //If we get here, the child didn't throw an exception var result = new ArrayList(); AddEnum(result, current_val as IEnumerable); AddEnum(result, rest); q.Enqueue(new Brunet.Util.Pair<object, bool>(result, false)); }