Exemplo n.º 1
0
        /**
         * Implements the IReplyHandler (also provides some light-weight statistics)
         */
        public bool HandleReply(ReqrepManager man, ReqrepManager.ReqrepType rt,
                                int mid, PType prot, MemBlock payload, ISender ret_path,
                                ReqrepManager.Statistics statistics, object state)
        {
            RpcRequestState rs = (RpcRequestState)state;
            //ISender target = rs.RpcTarget;
            Channel bq = rs.Results;

            if (bq != null)
            {
                object    data = AdrConverter.Deserialize(payload);
                RpcResult res  = new RpcResult(ret_path, data, statistics);
                //handle possible exception:
                try {
                    bq.Enqueue(res);
                    //Keep listening unless the queue is closed
                    return(!bq.Closed);
                }
                catch (System.InvalidOperationException) {
                    //The queue is closed, stop listening for more results:
                    return(false);
                }
            }
            else
            {
                //If they didn't even pass us a queue, I guess they didn't want to
                //listen too long
                return(false);
            }
        }
Exemplo n.º 2
0
    public override void Reduce(Channel q, object reduce_arg, 
                                  object current_result, RpcResult child_rpc) {

      bool done = false;
      ISender child_sender = child_rpc.ResultSender;
      //the following can throw an exception, will be handled by the framework
      object child_result = child_rpc.Result;
      

      //child result is a valid result
      if (current_result == null) {
        q.Enqueue(new Brunet.Collections.Pair<object, bool>(child_result, done));
        return;
      }
      

      ArrayList retval = current_result as ArrayList;
      IDictionary my_entry = (IDictionary) retval[0];
      my_entry["next_con"] = child_sender.ToUri();
      retval.AddRange((IList) child_result);
      
      if (LogEnabled) {
        ProtocolLog.Write(ProtocolLog.MapReduce, 
                          String.Format("{0}: {1}, reduce list count: {2}.", this.TaskName, _node.Address, retval.Count));
      }
      q.Enqueue(new Brunet.Collections.Pair<object, bool>(retval, done));
    }
Exemplo n.º 3
0
    public override void Reduce(Channel q, object reduce_arg, 
                                  object current_result, RpcResult child_rpc) {

      bool done = false;
      //ISender child_sender = child_rpc.ResultSender;
      //the following can throw an exception, will be handled by the framework
      object child_result = child_rpc.Result;
      
      //child result is a valid result
      if (current_result == null) {
        q.Enqueue(new Brunet.Collections.Pair<object, bool>(child_result, done));
        return;
      }
      
      IDictionary my_entry = current_result as IDictionary;
      IDictionary value = child_result as IDictionary;
      int max_height = (int) my_entry["height"];
      int count = (int) my_entry["count"];

      int y = (int) value["count"];
      my_entry["count"] = count + y;
      int z = (int) value["height"] + 1;
      if (z > max_height) {
        my_entry["height"] = z; 
      }
      q.Enqueue(new Brunet.Collections.Pair<object, bool>(my_entry, done));
    }
Exemplo n.º 4
0
        /**
         * When an error comes in, this handles it
         */
        public void HandleError(ReqrepManager man, int message_number,
                                ReqrepManager.ReqrepError err, ISender ret_path, object state)
        {
            Exception       x  = null;
            RpcRequestState rs = (RpcRequestState)state;
            Channel         bq = rs.Results;

            switch (err)
            {
            case ReqrepManager.ReqrepError.NoHandler:
                x = new AdrException(-32601, "No RPC Handler on remote host");
                break;

            case ReqrepManager.ReqrepError.HandlerFailure:
                x = new AdrException(-32603, "The remote RPC System had a problem");
                break;

            case ReqrepManager.ReqrepError.Timeout:
                //In this case we close the Channel:
                if (bq != null)
                {
                    bq.Close();
                }
                break;

            case ReqrepManager.ReqrepError.Send:
                //We had some problem sending, but ignore it for now
                break;
            }
            if (x != null && (bq != null))
            {
                RpcResult res = new RpcResult(ret_path, x);
                bq.Enqueue(res);
            }
        }
Exemplo n.º 5
0
 public override void Reduce(Channel q, object reduce_arg, object current_val, RpcResult child_r) {
   var rest = child_r.Result as IEnumerable;
   //If we get here, the child didn't throw an exception
   var result = new ArrayList();
   AddEnum(result, current_val as IEnumerable);
   AddEnum(result, rest);
   q.Enqueue(new Pair<object, bool>(result, false));
 }
Exemplo n.º 6
0
    /**
     * Reduce method
     * @param reduce_arg argument for reduce
     * @param current_result result of current map 
     * @param child_rpc results from children 
     * @param done if done is true, stop reducing and return result
     * return table of hop count, number of success, tree depth
     */
    public override void Reduce(Channel q, object reduce_arg, 
                                  object current_result, RpcResult child_rpc) {

      bool done = false;
      //ISender child_sender = child_rpc.ResultSender;
      //the following can throw an exception, will be handled by the framework
      object child_result = UNSET;
      try {
        child_result = child_rpc.Result;
      }
      catch(Exception x) {
        /*
        if (x.Equals("Child did not return") ) {
          //there might be at least one timeout from children.
	  //Start boundedcasting again
          IDictionary ht;
          string task_name = "Brunet.Services.Deetoo.MapReduceCache;
	  ht["task_name"] = task_name 
	  IDictionary map_arg;
	  IDictionary gen_arg;

	  = (IDictionary) args[0];
          MapReduceArgs mr_args = new MapReduceArgs(ht);
          //string task_name = mr_args.TaskName;
          MapReduceTask task;
          if (_name_to_task.TryGetValue(task_name, out task)) {
            MapReduceComputation mr = new MapReduceComputation(_node, req_state, task, mr_args);
            mr.Start();
          }
	}
        */
      } 
      if( child_result == UNSET ) {
        //The child threw an exception
        q.Enqueue(new Brunet.Collections.Pair<object, bool> (current_result, done));
        return;
      } 
      //child result is a valid result
      if (current_result == null) {
        q.Enqueue(new Brunet.Collections.Pair<object, bool> (child_result, done));
        return;
      }
      //else combine them: 
      IDictionary my_entry = (IDictionary)current_result;
      IDictionary value = (IDictionary)child_result;
      int max_height = (int) my_entry["height"];
      int count = (int) my_entry["count"];
      my_entry["success"] = (int) my_entry["success"] + (int) value["success"];
      int y = (int) value["count"];
      my_entry["count"] = count + y;
      int z = (int) value["height"] + 1;
      if (z > max_height) {
        my_entry["height"] = z; 
      }
      q.Enqueue(new Brunet.Collections.Pair<object, bool>(my_entry,done));
    }
Exemplo n.º 7
0
 /**
  * When an error comes in, this handles it
  */
 public void HandleError(ReqrepManager man, int message_number,
                  ReqrepManager.ReqrepError err, ISender ret_path, object state)
 {
   Exception x = null;
   RpcRequestState rs = (RpcRequestState) state;
   Channel bq = rs.Results;
   switch(err) {
       case ReqrepManager.ReqrepError.NoHandler:
         x = new AdrException(-32601, "No RPC Handler on remote host");
         break;
       case ReqrepManager.ReqrepError.HandlerFailure:
         x = new AdrException(-32603, "The remote RPC System had a problem");
         break;
       case ReqrepManager.ReqrepError.Timeout:
         //In this case we close the Channel:
         if( bq != null ) { bq.Close(); }
         break;
       case ReqrepManager.ReqrepError.Send:
         if( rs.RpcTarget is Edge ) { 
           Edge e = (Edge)rs.RpcTarget;
           //This definitely won't get any more responses:
           if( e.IsClosed && bq != null ) { bq.Close(); } 
         }
         //We had some problem sending, but ignore it for now
         break;
   }
   if( x != null && (bq != null) ) {
     RpcResult res = new RpcResult(ret_path, x);
     bq.Enqueue(res);
   }
 }
Exemplo n.º 8
0
  /**
   * Implements the IReplyHandler (also provides some light-weight statistics)
   */
  public bool HandleReply(ReqrepManager man, ReqrepManager.ReqrepType rt,
			  int mid, PType prot, MemBlock payload, ISender ret_path,
			  ReqrepManager.Statistics statistics, object state)
  {
    RpcRequestState rs = (RpcRequestState) state;
    //ISender target = rs.RpcTarget;
    Channel bq = rs.Results;
    if( bq != null ) {
      object data = AdrConverter.Deserialize(payload);
      RpcResult res = new RpcResult(ret_path, data, statistics);
      //handle possible exception:
      try {
        bq.Enqueue(res);
        //Keep listening unless the queue is closed
        return (!bq.Closed);
      }
      catch(System.InvalidOperationException) {
        //The queue is closed, stop listening for more results:
        return false;
      }
    }
    else {
      //If they didn't even pass us a queue, I guess they didn't want to
      //listen too long
      return false;
    }
  }
Exemplo n.º 9
0
    /// <summary>Process a new results and determine whether or not to execute
    /// more tests on this node and the neighbors it returned.</summary>
    protected void ProcessResults(Address addr, RpcResult result)
    {
      ResultState rs = null;
      lock(_sync) {
        rs = _results[addr];
      }

      if(result == null || result.Statistics.SendCount != 1) {
        rs.RTTs[rs.CurrentAttempt] = TimeSpan.MinValue;
        rs.Hops[rs.CurrentAttempt] = -1;
      } else {
        rs.RTTs[rs.CurrentAttempt] = DateTime.UtcNow - rs.StartTime;
        rs.Hops[rs.CurrentAttempt] = (result.ResultSender as AHSender).HopsTaken;
      }

      rs.CurrentAttempt++;

      List<ResultState> to_invoke = new List<ResultState>();
      bool finished = false;

      Hashtable data = null;
      if(result != null) {
        data = result.Result as Hashtable;
      }

      lock(_sync) {
        if(rs.CurrentAttempt == 3) {
          _outstanding.Remove(addr);
        } else {
          to_invoke.Add(rs);
        }
        if(data != null) {
          foreach(string peer in data.Values) {
            Address peer_addr = AddressParser.Parse(peer);
            if(_results.ContainsKey(peer_addr)) {
              continue;
            }
            ResultState nrs = new ResultState(peer_addr);
            to_invoke.Add(nrs);
            _results[peer_addr] = nrs;
            _outstanding[peer_addr] = true;
          }
        }
        if(_outstanding.Count == 0) {
          finished = true;
        }
      }

      if(finished) {
        ThreadPool.QueueUserWorkItem(delegate(object o) { Finished(); } );
      }

      foreach(ResultState nrs in to_invoke) {
        nrs.StartTime = DateTime.UtcNow;
        Invoke(nrs.Target);
      }
    }
Exemplo n.º 10
0
    private void AddStuffToBQAndClose(object ostate) {
      object[] state = (object[])ostate;
      BlockingQueue bq = (BlockingQueue)state[0];
      ISender target = (ISender)state[1];

      foreach (object o in this.CurrentInvokeState.RetValues) {
        if (CurrentInvokeState.EnqueueIntervalInMillisec >= 0) {
          Thread.Sleep(CurrentInvokeState.EnqueueIntervalInMillisec);
        }
        RpcResult rs = new RpcResult(target, o);
        bq.Enqueue(rs);
      }

      if (CurrentInvokeState.IntervalBetweenLastEnqueueAndClose >= 0) {
        Thread.Sleep(CurrentInvokeState.IntervalBetweenLastEnqueueAndClose); 
      }
      bq.Close();
    }
Exemplo n.º 11
0
 /** 
  * reduce function.  This reduces the local and children results into one.
  * This is also the error handling function.  Any exceptions must be
  * handled by this method, if not, the computation stops immediately and
  * sends the exception back up the tree.
  * 
  * @param q the Channel into which a Pair<object, bool> is enqueued,
  * if the second item is true, we stop querying nodes
  * @param reduce_arg arguments for the reduce
  * @param current_result accumulated result of reductions
  * @param child_rpc result from child computation
  */
 public virtual void Reduce(Channel q, object reduce_arg, object current_result, RpcResult child_rpc) {
   throw new NotImplementedException();
 }
Exemplo n.º 12
0
 protected void ChildCallback(object cq, EventArgs arg) {
   RpcResult child_r;
   Channel child_q = (Channel)cq;
   MapReduceInfo mri = (MapReduceInfo)child_q.State;
   if (LogEnabled) {
     ProtocolLog.Write(ProtocolLog.MapReduce,        
                       String.Format("MapReduce: {0}, handling child result from: {1}.", _node.Address, mri.Sender.ToUri()));
   }
   if( child_q.Count == 0 ) {
     child_r = new RpcResult(mri.Sender, new AdrException(-32000, "Child did not return"));
   }
   else {
     child_r = (RpcResult)child_q.Dequeue();
   }
   Reduce(child_r);
 }
Exemplo n.º 13
0
 /**
  * Reduce method
  * @param reduce_args argument for reduce
  * @param current_result result of current map 
  * @param child_rpc results from children 
  * return table of hop count, tree depth, and query result
  */  
 public override void Reduce(Channel q, object reduce_args, 
                               object current_result, RpcResult child_rpc) {
   object child_result = null;
   child_result = child_rpc.Result;
   Converter<object,HitCombiner> hcf = null;
   IList args = (IList)reduce_args;
   string query_type = (string)(args[0]);
   object hc_arg = args[1];
   if( _mut.State.HCFact.TryGetValue(query_type, out hcf) ) {
     HitCombiner hc = hcf(hc_arg);
     if( current_result == null ) {
       IDictionary child_val = (IDictionary)child_result;
       //Initially, we are empty:
       var res = hc.Combine(new ArrayList(), (IList)child_val["query_result"]);
       var ret_val = new ListDictionary();
       ret_val["height"] = child_val["height"];
       ret_val["count"] = child_val["count"];
       ret_val["query_result"] = res.First;
       q.Enqueue(new Brunet.Collections.Pair<object, bool>(ret_val, res.Second));
     }
     else {
       IDictionary current_val = (IDictionary)current_result;
       IDictionary child_val = (IDictionary)child_result;
       
       var ret_val = new ListDictionary();
       ret_val["count"] = (int)current_val["count"] + (int)child_val["count"];
       ret_val["height"] = Math.Max((int)current_val["height"],
                                   1 + (int)child_val["height"]);
       
       var res = hc.Combine((IList)current_val["query_result"],
                              (IList)child_val["query_result"]);
       ret_val["query_result"] = res.First;
       q.Enqueue(new Brunet.Collections.Pair<object, bool>(ret_val, res.Second));
     }
   }
   else {
     q.Enqueue(new AdrException(-32608, "This query type {0} is not supported." + query_type) );
   }
 }
Exemplo n.º 14
0
 protected void Reduce(RpcResult child_r) {
   //The usual transactional bit:
   State state = _state;
   State old_state;
   State new_state;
   do {
     old_state = state;
     new_state = old_state.AddChildResult(child_r);
     state = Interlocked.CompareExchange<State>(ref _state, new_state, old_state);
   }
   while( state != old_state);
   //If we need to start a new reduce, it's the latest value:  
   TryNextReduce(new_state, old_state, child_r, false);
 }
Exemplo n.º 15
0
 /**
  * @return true if we successfully started the next reduce
  */
 protected bool TryNextReduce(State new_s, State old_s, RpcResult v, bool cont) {
   if( new_s.Done ) {
     SendResult( new_s.ReduceResult );
     return false;
   }
   bool start_red = new_s.Reducing && (cont || (false == old_s.Reducing));
   if (LogEnabled) {
     ProtocolLog.Write(ProtocolLog.MapReduce,        
                       String.Format("MapReduce: {0}, TryNextReduce: {1}.",
                       _node.Address, start_red));
   }
   if( start_red ) {
     Channel r_chan = new Channel(1, v);
     r_chan.CloseEvent += this.ReduceHandler;
     object startval = new_s.ReduceResult == State.DEFAULT_OBJ ? null : new_s.ReduceResult;
     if (LogEnabled) {
       ProtocolLog.Write(ProtocolLog.MapReduce,        
                       String.Format("MapReduce: {0} abt to Reduce({1},{2},{3})",
                         _node.Address, _mr_args.ReduceArg, startval, v));
     }
     try {
       _mr_task.Reduce(r_chan, _mr_args.ReduceArg, startval, v);
     }
     catch(Exception x) {
       //Reduce is where we do error handling, if that doesn't work, oh well:
       if (LogEnabled) {
         ProtocolLog.Write(ProtocolLog.MapReduce,        
                       String.Format("MapReduce: {0}, reduce threw: {1}.", _node.Address, x));
       }
       SendResult(x);
       return false;
     }
   }
   return start_red;
 }
Exemplo n.º 16
0
 /*
  * We always reduce our map call first, then children.
  */
 public State AddChildResult(RpcResult child) {
   if( Reducing ) {
     //We need to add this to pending
     var pend = new ImmutableList<RpcResult>(child, Pending);
     return new State(MapResult, Tree, ChildReductions, ReduceResult, Done, pend, true); 
   }
   else if( MapResult == DEFAULT_OBJ ) {
     /*
      * We are not reducing AND we have not finished the first
      * Reduce, that means we are waiting for the Map result.
      */
     var pend = new ImmutableList<RpcResult>(child, Pending);
     return new State(MapResult, Tree, ChildReductions, ReduceResult, Done, pend, false); 
   }
   else {
     /*
      * In this case, we are not already reducing, and we have reduced
      * the MapResult
      */
     return new State(MapResult, Tree, ChildReductions, ReduceResult, Done, ImmutableList<RpcResult>.Empty, true); 
   }
 }
Exemplo n.º 17
0
    /**
     * This dispatches the particular methods this class provides.
     * Currently, the only invokable method is:
     * "Start". 
     */
    public void HandleRpc(ISender caller, string method, IList args, object req_state) {
      int part_idx = method.IndexOf(':');
      if( part_idx == -1 ) {
        if (method == "Start") {
          IDictionary ht = (IDictionary) args[0];
          MapReduceArgs mr_args = new MapReduceArgs(ht);
          string task_name = mr_args.TaskName;
          MapReduceTask task;
          if (_name_to_task.TryGetValue(task_name, out task)) {
            MapReduceComputation mr = new MapReduceComputation(_node, req_state, task, mr_args);
            mr.Start();
          } 
          else {
            throw new AdrException(-32608, "No mapreduce task with name: " + task_name);          
          }
        }
        else if( method == "AddHandler" ) {
          //Make sure this is local:
          ISender tmp_call = caller;
          bool islocal = tmp_call is Node;
          while(!islocal && tmp_call is IWrappingSender) {
            tmp_call =  ((IWrappingSender)tmp_call).WrappedSender;
            islocal = tmp_call is Node;
          }
          if( !islocal ) {
            throw new AdrException(-32601, "AddHandler only valid for local callers");
          }
          SubscribeTask(new RpcMapReduceTask(_node, (IDictionary)args[0]));
          _rpc.SendResult(req_state, null);
        }
        else {
          throw new AdrException(-32601, "No Handler for method: " + method);
        }
      }
      else {
        //This is a reference to a specific part of a task:
        string part = method.Substring(0, part_idx);
        string task_name = method.Substring(part_idx + 1);
        MapReduceTask task;
        if(false == _name_to_task.TryGetValue(task_name, out task)) {
          throw new AdrException(-32608, "No mapreduce task with name: " + task_name);          
        }
        if( part == "tree" ) {
          var mra = new MapReduceArgs((IDictionary)args[0]);

          var tree_res = new Channel(1, req_state);
          tree_res.CloseEvent += this.HandleTree;
          task.GenerateTree(tree_res, mra);
        }
        else if( part == "reduce" ) {
          //Prepare the RpcResult:
          var rres_d = (IDictionary)args[2];
          ISender send = SenderFactory.CreateInstance(_node, (string)rres_d["sender"]);
          var rres = new RpcResult(send, rres_d["result"]);
          
          Channel reduce_res = new Channel(1, req_state);
          reduce_res.CloseEvent += this.HandleReduce;
          task.Reduce(reduce_res, args[0], args[1], rres);
        }
        else if( part == "map" ) {
          Channel map_res = new Channel(1, req_state);
          map_res.CloseEvent += this.HandleMap;
          task.Map(map_res, args[0]);
        }
        else {
          throw new AdrException(-32608,
              String.Format("No mapreduce task({0}) part with name: {1}", task_name, part));          
        }
      }
    }
Exemplo n.º 18
0
 public override void Reduce(Channel q, object reduce_arg, object current_result, RpcResult child_rpc) {
   Channel result = new Channel(1, q);
   result.CloseEvent += this.ReduceHandler;
   var childrpc_ht = new Hashtable();
   ISender rsend = child_rpc.ResultSender;
   childrpc_ht["sender"] = rsend != null ? rsend.ToUri() : "sender:localnode";
   try {
     //If this is an exception, this will throw
     childrpc_ht["result"] = child_rpc.Result; 
   }
   catch(Exception x) {
     childrpc_ht["result"] = x;
   }
   _node.Rpc.Invoke(_reduce.First, result, _reduce.Second, reduce_arg, current_result, childrpc_ht);
 }