public override object Reduce(object reduce_arg,
object current_result, RpcResult child_rpc,
out bool done) {
done = false;
ISender child_sender = child_rpc.ResultSender;
//the following can throw an exception, will be handled by the framework
object child_result = child_rpc.Result;
//child result is a valid result
if (current_result == null) {
return child_result;
}
ArrayList retval = current_result as ArrayList;
IDictionary my_entry = (IDictionary) retval[0];
Edge e = child_sender as Edge;
my_entry["next_con"] = e.ToString();
retval.AddRange((IList) child_result);
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("{0}: {1}, reduce list count: {2}.", this.TaskName, _node.Address, retval.Count));
}
return retval;
}
public override void Reduce(Channel q, object reduce_arg,
object current_result, RpcResult child_rpc) {
bool done = false;
ISender child_sender = child_rpc.ResultSender;
//the following can throw an exception, will be handled by the framework
object child_result = child_rpc.Result;
//child result is a valid result
if (current_result == null) {
q.Enqueue(new Brunet.Util.Pair<object, bool>(child_result, done));
return;
}
ArrayList retval = current_result as ArrayList;
IDictionary my_entry = (IDictionary) retval[0];
my_entry["next_con"] = child_sender.ToUri();
retval.AddRange((IList) child_result);
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("{0}: {1}, reduce list count: {2}.", this.TaskName, _node.Address, retval.Count));
}
q.Enqueue(new Brunet.Util.Pair<object, bool>(retval, done));
}
public override object Reduce(object reduce_arg,
object current_result, RpcResult child_rpc,
out bool done) {
done = false;
//ISender child_sender = child_rpc.ResultSender;
//the following can throw an exception, will be handled by the framework
object child_result = child_rpc.Result;
//child result is a valid result
if (current_result == null) {
return child_result;
}
IDictionary my_entry = current_result as IDictionary;
IDictionary value = child_result as IDictionary;
int max_height = (int) my_entry["height"];
int count = (int) my_entry["count"];
int y = (int) value["count"];
my_entry["count"] = count + y;
int z = (int) value["height"] + 1;
if (z > max_height) {
my_entry["height"] = z;
}
return my_entry;
}
public override void Reduce(Channel q, object reduce_arg,
object current_result, RpcResult child_rpc) {
bool done = false;
//ISender child_sender = child_rpc.ResultSender;
//the following can throw an exception, will be handled by the framework
object child_result = child_rpc.Result;
//child result is a valid result
if (current_result == null) {
q.Enqueue(new Brunet.Util.Pair<object, bool>(child_result, done));
return;
}
IDictionary my_entry = current_result as IDictionary;
IDictionary value = child_result as IDictionary;
int max_height = (int) my_entry["height"];
int count = (int) my_entry["count"];
int y = (int) value["count"];
my_entry["count"] = count + y;
int z = (int) value["height"] + 1;
if (z > max_height) {
my_entry["height"] = z;
}
q.Enqueue(new Brunet.Util.Pair<object, bool>(my_entry, done));
}
/**
* When an error comes in, this handles it
*/
public void HandleError(ReqrepManager man, int message_number,
ReqrepManager.ReqrepError err, ISender ret_path, object state)
{
Exception x = null;
RpcRequestState rs = (RpcRequestState) state;
Channel bq = rs.Results;
switch(err) {
case ReqrepManager.ReqrepError.NoHandler:
x = new AdrException(-32601, "No RPC Handler on remote host");
break;
case ReqrepManager.ReqrepError.HandlerFailure:
x = new AdrException(-32603, "The remote RPC System had a problem");
break;
case ReqrepManager.ReqrepError.Timeout:
//In this case we close the Channel:
if( bq != null ) { bq.Close(); }
break;
case ReqrepManager.ReqrepError.Send:
if( rs.RpcTarget is Edge ) {
Edge e = (Edge)rs.RpcTarget;
//This definitely won't get any more responses:
if( e.IsClosed && bq != null ) { bq.Close(); }
}
//We had some problem sending, but ignore it for now
break;
}
if( x != null && (bq != null) ) {
RpcResult res = new RpcResult(ret_path, x);
bq.Enqueue(res);
}
}
/**
* Implements the IReplyHandler (also provides some light-weight statistics)
*/
public bool HandleReply(ReqrepManager man, ReqrepManager.ReqrepType rt,
int mid, PType prot, MemBlock payload, ISender ret_path,
ReqrepManager.Statistics statistics, object state)
{
RpcRequestState rs = (RpcRequestState) state;
//ISender target = rs.RpcTarget;
Channel bq = rs.Results;
if( bq != null ) {
object data = AdrConverter.Deserialize(payload);
RpcResult res = new RpcResult(ret_path, data, statistics);
bq.Enqueue(res);
//Keep listening unless the queue is closed
return (!bq.Closed);
}
else {
//If they didn't even pass us a queue, I guess they didn't want to
//listen too long
return false;
}
}
protected void ChildCallback(object cq, EventArgs arg) {
RpcResult child_r;
Channel child_q = (Channel)cq;
MapReduceInfo mri = (MapReduceInfo)child_q.State;
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("MapReduce: {0}, handling child result from: {1}.", _node.Address, mri.Sender.ToUri()));
}
if( child_q.Count == 0 ) {
child_r = new RpcResult(mri.Sender, new AdrException(-32000, "Child did not return"));
}
else {
child_r = (RpcResult)child_q.Dequeue();
}
Reduce(child_r);
}
protected void Reduce(RpcResult child_r) {
//The usual transactional bit:
State state = _state;
State old_state;
State new_state;
do {
old_state = state;
new_state = old_state.AddChildResult(child_r);
state = Interlocked.CompareExchange<State>(ref _state, new_state, old_state);
}
while( state != old_state);
//If we need to start a new reduce, it's the latest value:
TryNextReduce(new_state, old_state, child_r, false);
}
/**
* @return true if we successfully started the next reduce
*/
protected bool TryNextReduce(State new_s, State old_s, RpcResult v, bool cont) {
if( new_s.Done ) {
SendResult( new_s.ReduceResult );
return false;
}
bool start_red = new_s.Reducing && (cont || (false == old_s.Reducing));
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("MapReduce: {0}, TryNextReduce: {1}.",
_node.Address, start_red));
}
if( start_red ) {
Channel r_chan = new Channel(1, v);
r_chan.CloseEvent += this.ReduceHandler;
object startval = new_s.ReduceResult == State.DEFAULT_OBJ ? null : new_s.ReduceResult;
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("MapReduce: {0} abt to Reduce({1},{2},{3})",
_node.Address, _mr_args.ReduceArg, startval, v));
}
try {
_mr_task.Reduce(r_chan, _mr_args.ReduceArg, startval, v);
}
catch(Exception x) {
//Reduce is where we do error handling, if that doesn't work, oh well:
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.MapReduce,
String.Format("MapReduce: {0}, reduce threw: {1}.", _node.Address, x));
}
SendResult(x);
return false;
}
}
return start_red;
}
/*
* We always reduce our map call first, then children.
*/
public State AddChildResult(RpcResult child) {
if( Reducing ) {
//We need to add this to pending
var pend = new ImmutableList<RpcResult>(child, Pending);
return new State(MapResult, Tree, ChildReductions, ReduceResult, Done, pend, true);
}
else if( MapResult == DEFAULT_OBJ ) {
/*
* We are not reducing AND we have not finished the first
* Reduce, that means we are waiting for the Map result.
*/
var pend = new ImmutableList<RpcResult>(child, Pending);
return new State(MapResult, Tree, ChildReductions, ReduceResult, Done, pend, false);
}
else {
/*
* In this case, we are not already reducing, and we have reduced
* the MapResult
*/
return new State(MapResult, Tree, ChildReductions, ReduceResult, Done, ImmutableList<RpcResult>.Empty, true);
}
}
/**
* This dispatches the particular methods this class provides.
* Currently, the only invokable method is:
* "Start".
*/
public void HandleRpc(ISender caller, string method, IList args, object req_state) {
int part_idx = method.IndexOf(':');
if( part_idx == -1 ) {
if (method == "Start") {
IDictionary ht = (IDictionary) args[0];
MapReduceArgs mr_args = new MapReduceArgs(ht);
string task_name = mr_args.TaskName;
MapReduceTask task;
if (_name_to_task.TryGetValue(task_name, out task)) {
MapReduceComputation mr = new MapReduceComputation(_node, req_state, task, mr_args);
mr.Start();
}
else {
throw new AdrException(-32608, "No mapreduce task with name: " + task_name);
}
}
else if( method == "AddHandler" ) {
//Make sure this is local:
ISender tmp_call = caller;
bool islocal = tmp_call is Node;
while(!islocal && tmp_call is IWrappingSender) {
tmp_call = ((IWrappingSender)tmp_call).WrappedSender;
islocal = tmp_call is Node;
}
if( !islocal ) {
throw new AdrException(-32601, "AddHandler only valid for local callers");
}
SubscribeTask(new RpcMapReduceTask(_node, (IDictionary)args[0]));
_rpc.SendResult(req_state, null);
}
else {
throw new AdrException(-32601, "No Handler for method: " + method);
}
}
else {
//This is a reference to a specific part of a task:
string part = method.Substring(0, part_idx);
string task_name = method.Substring(part_idx + 1);
MapReduceTask task;
if(false == _name_to_task.TryGetValue(task_name, out task)) {
throw new AdrException(-32608, "No mapreduce task with name: " + task_name);
}
if( part == "tree" ) {
var mra = new MapReduceArgs((IDictionary)args[0]);
var tree_res = new Channel(1, req_state);
tree_res.CloseEvent += this.HandleTree;
task.GenerateTree(tree_res, mra);
}
else if( part == "reduce" ) {
//Prepare the RpcResult:
var rres_d = (IDictionary)args[2];
ISender send = SenderFactory.CreateInstance(_node, (string)rres_d["sender"]);
var rres = new RpcResult(send, rres_d["result"]);
Channel reduce_res = new Channel(1, req_state);
reduce_res.CloseEvent += this.HandleReduce;
task.Reduce(reduce_res, args[0], args[1], rres);
}
else if( part == "map" ) {
Channel map_res = new Channel(1, req_state);
map_res.CloseEvent += this.HandleMap;
task.Map(map_res, args[0]);
}
else {
throw new AdrException(-32608,
String.Format("No mapreduce task({0}) part with name: {1}", task_name, part));
}
}
}
public override void Reduce(Channel q, object reduce_arg, object current_result, RpcResult child_rpc) {
Channel result = new Channel(1, q);
result.CloseEvent += this.ReduceHandler;
var childrpc_ht = new Hashtable();
ISender rsend = child_rpc.ResultSender;
childrpc_ht["sender"] = rsend != null ? rsend.ToUri() : "sender:localnode";
try {
//If this is an exception, this will throw
childrpc_ht["result"] = child_rpc.Result;
}
catch(Exception x) {
childrpc_ht["result"] = x;
}
_node.Rpc.Invoke(_reduce.First, result, _reduce.Second, reduce_arg, current_result, childrpc_ht);
}
/**
* reduce function. This reduces the local and children results into one.
* This is also the error handling function. Any exceptions must be
* handled by this method, if not, the computation stops immediately and
* sends the exception back up the tree.
*
* @param q the Channel into which a Brunet.Util.Pair<object, bool> is enqueued,
* if the second item is true, we stop querying nodes
* @param reduce_arg arguments for the reduce
* @param current_result accumulated result of reductions
* @param child_rpc result from child computation
*/
public virtual void Reduce(Channel q, object reduce_arg, object current_result, RpcResult child_rpc) {
throw new NotImplementedException();
}
/** * reduce function. * @param reduce_arg arguments for the reduce * @param current_result accumulated result of reductions * @param child_rpc result from child computation * @param done out parameter (is the stopping criteria met) * @param child_results hashtable containing results from each child */ public abstract object Reduce(object reduce_arg, object current_result, RpcResult child_rpc, out bool done);
public override void Reduce(Channel q, object reduce_arg, object current_val, RpcResult child_r) {
var rest = child_r.Result as IEnumerable;
//If we get here, the child didn't throw an exception
var result = new ArrayList();
AddEnum(result, current_val as IEnumerable);
AddEnum(result, rest);
q.Enqueue(new Brunet.Util.Pair<object, bool>(result, false));
}
