public override MapReduceInfo[] GenerateTree(MapReduceArgs mr_args) {
      object gen_arg = mr_args.GenArg;
      Log("{0}: {1}, greedy generator called, arg: {2}.", 
          this.TaskName, _node.Address, gen_arg);
      string address = gen_arg as string;
      AHAddress a =  (AHAddress) AddressParser.Parse(address);
      ArrayList retval = new ArrayList();
      ConnectionTable tab = _node.ConnectionTable;
      ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
      Connection next_closest = structs.GetNearestTo((AHAddress) _node.Address, a);
      if (next_closest != null) {
        int timeout = -1;
        if (mr_args.MillisecTimeout >= 0) { //only if it is a valid timeout
          timeout = (int) (TIMEOUT_FRACTION*mr_args.MillisecTimeout);
        }

        MapReduceInfo mr_info = new MapReduceInfo( (ISender) next_closest.Edge,
                                                   new MapReduceArgs(this.TaskName,//arguments do not change much
                                                                     mr_args.MapArg,
                                                                     mr_args.GenArg,
                                                                     mr_args.ReduceArg,
                                                                     timeout));//only timeout changes


        retval.Add(mr_info);
      }
      
      Log("{0}: {1}, greedy generator returning: {2} senders.", 
          this.TaskName, _node.Address, retval.Count);
      return (MapReduceInfo[]) retval.ToArray(typeof(MapReduceInfo));
    }
Exemplo n.º 2
0
 public override MapReduceInfo[] GenerateTree(MapReduceArgs mr_args) {
   object gen_arg = mr_args.GenArg;
   Log("{0}: {1}, greedy generator called, arg: {2}.", 
       this.TaskName, _node.Address, gen_arg);
   string address = gen_arg as string;
   AHAddress a =  (AHAddress) AddressParser.Parse(address);
   ArrayList retval = new ArrayList();
   ConnectionTable tab = _node.ConnectionTable;
   ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
   Connection next_closest = structs.GetNearestTo((AHAddress) _node.Address, a);
   if (next_closest != null) {
     MapReduceInfo mr_info = new MapReduceInfo( (ISender) next_closest.Edge,
                                                mr_args); //arguments do not change at all
     retval.Add(mr_info);
   }
   
   Log("{0}: {1}, greedy generator returning: {2} senders.", 
       this.TaskName, _node.Address, retval.Count);
   return (MapReduceInfo[]) retval.ToArray(typeof(MapReduceInfo));
 }
Exemplo n.º 3
0
 /*
  * This cannot change our Reducing state.
  */
 public State UpdateTree(MapReduceInfo[] tree) {
   if( Tree != null ) {
     throw new Exception("MapReduce: Tree already set");
   }
   bool done = Done;
   if( ReduceResult != DEFAULT_OBJ ) {
     /*
      * We have already reduced the map result, we are done, ONLY if
      * the tree is empty:
      */
     done = done || (tree.Length == 0);
   }
   else {
     //We haven't yet reduced the map result
     //Done should be false
   }
   return new State(MapResult, tree, ChildReductions, ReduceResult, done, Pending, Reducing);
 }
Exemplo n.º 4
0
 protected State(object map, MapReduceInfo[] tree, int cred, object reduce, bool done, ImmutableList<RpcResult> pend, bool reducing) {
   MapResult = map;
   Tree = tree;
   ChildReductions = cred;
   ReduceResult = reduce;
   Done = done;
   Pending = pend;
   Reducing = reducing;
 }
Exemplo n.º 5
0
    /** Starts the computation. */
    public void Start() {
      //invoke map
      try {
        _map_result = _mr_task.Map(_mr_args.MapArg);
      } 
      catch(Exception x) {
        if (ProtocolLog.MapReduce.Enabled) {
          ProtocolLog.Write(ProtocolLog.MapReduce, 
                            String.Format("MapReduce: {0}, map exception: {1}.", _node.Address, x));        
        }
        _finished = true;
        SendResult(x);
        return;
      }

      if (LogEnabled) {
        ProtocolLog.Write(ProtocolLog.MapReduce,
                          String.Format("MapReduce: {0}, map result: {1}.", _node.Address, _map_result));
      }

      //do an initial reduction and see if we can terminate
      try {
        bool done; //out parameter
        _reduce_result = _mr_task.Reduce(_mr_args.ReduceArg, null, new RpcResult(null, _map_result), out done);

        if (LogEnabled) {
          ProtocolLog.Write(ProtocolLog.MapReduce,
                            String.Format("MapReduce: {0}, initial reduce result: {1}.", _node.Address, _reduce_result));
        }

        if (done) {
          _finished = true;
          SendResult(_reduce_result);
          return;
        }
      } catch(Exception x) {
        if (ProtocolLog.MapReduce.Enabled) {
          ProtocolLog.Write(ProtocolLog.MapReduce, 
                            String.Format("MapReduce: {0}, initial reduce exception: {1}.", _node.Address, x));
        }
        _finished = true;
        SendResult(x);
        return;
      }

      //compute the list of child targets
      MapReduceInfo[] child_mr_info = null;
      try {
        child_mr_info = _mr_task.GenerateTree(_mr_args);
      } catch (Exception x) {
        if (ProtocolLog.MapReduce.Enabled) {
          child_mr_info = new MapReduceInfo[0];
          ProtocolLog.Write(ProtocolLog.MapReduce,         
                            String.Format("MapReduce: {0}, generate tree exception: {1}.", _node.Address, x));
        }
      }
      
      if (LogEnabled) {
        ProtocolLog.Write(ProtocolLog.MapReduce,
                          String.Format("MapReduce: {0}, child senders count: {1}.", _node.Address, child_mr_info.Length));
      }

      if (child_mr_info.Length > 0) {
        foreach ( MapReduceInfo mr_info in child_mr_info) {
          Channel child_q = new Channel(1);
          //so far this is thread-safe
          _queue_to_child[child_q] = mr_info;
        }
        
        foreach (DictionaryEntry de in _queue_to_child) {
          Channel child_q = (Channel) de.Key;
          MapReduceInfo mr_info = (MapReduceInfo) de.Value;

          //the following will prevent the current object from going out of scope. 
          child_q.EnqueueEvent += new EventHandler(ChildCallback);
          try {
            _rpc.Invoke(mr_info.Sender, child_q,  "mapreduce.Start", mr_info.Args.ToHashtable());
          } catch(Exception) {
            ChildCallback(child_q, null);
          }
        }
      } else {
        // did not generate any child computations, return rightaway
        _finished = true;
        SendResult(_reduce_result);
        return;
      }
    }
    /**
     * When a node is out of the range, this method is called.
     * This method tries to find the nearest node to the middle of range using greedty algorithm.
     * return list of MapReduceInfo
     */
    private ArrayList GenerateTreeOutRange(AHAddress start, AHAddress end, MapReduceArgs mr_args, int timeout) {
      ArrayList retval = new ArrayList();
      BigInteger up = start.ToBigInteger();
      BigInteger down = end.ToBigInteger();
      BigInteger mid_range = (up + down) /2;
      if (mid_range % 2 == 1) {mid_range = mid_range -1; }
	AHAddress mid_addr = new AHAddress(mid_range);
	if (!mid_addr.IsBetweenFromLeft(start, end) ) {
          mid_range += Address.Half;
	  mid_addr = new AHAddress(mid_range);
      }
      ArrayList gen_arg = new ArrayList();
      if (NextGreedyClosest(mid_addr) != null ) {
        AHGreedySender ags = new AHGreedySender(_node, mid_addr);
	string start_range = start.ToString();
	string end_range = end.ToString();
	gen_arg.Add(start_range);
	gen_arg.Add(end_range);
        MapReduceInfo mr_info = new MapReduceInfo( (ISender) ags,
				                new MapReduceArgs(this.TaskName,
							          mr_args.MapArg,
								  gen_arg,
                                                                  mr_args.ReduceArg,
								  timeout));
	Log("{0}: {1}, out of range, moving to the closest node to mid_range: {2} to target node, range start: {3}, range end: {4}",
			  this.TaskName, _node.Address, mid_addr, start, end);
	retval.Add(mr_info);
      }
      else  {
        // cannot find a node in the range. 
      }
      return retval;
    }
    /**
     * Generate tree within the range.
     * return list of MapReduceInfo
     */
    private ArrayList GenerateTreeInRange(AHAddress this_addr, AHAddress start, AHAddress end, List<Connection> cons, bool left, MapReduceArgs mr_args, int timeout) {
      //Divide the range and trigger bounded broadcasting again in divided range starting with neighbor.
      //Deivided ranges are (start, n_1), (n_1, n_2), ... , (n_m, end)
      ArrayList retval = new ArrayList();
      if (cons.Count != 0) //make sure if connection list is not empth!
      {
        //con_list is sorted.
	AHAddress last = this_addr;
	//the first element of cons is the nearest.
        for (int i = 0; i < cons.Count; i++) {
	  Connection next_c = (Connection)cons[i];
	  AHAddress next_addr = (AHAddress)next_c.Address;
	  ISender sender = (ISender) next_c.Edge;
	  string front = last.ToString();
	  string back = next_addr.ToString();
	  string rg_start = start.ToString();
	  string rg_end = end.ToString();
          ArrayList gen_arg = new ArrayList();
	  if (i==cons.Count -1) {  // The last bit
            if (left) {
	      // the left farthest neighbor 
	      gen_arg.Add(front);
	      gen_arg.Add(rg_end);
	    }
	    else {
	      // the right farthest neighbor
              gen_arg.Add(rg_start);
	      gen_arg.Add(front);
	    }
	  }
	  else {
	    if (left) { //left connections
              gen_arg.Add(front);
	      gen_arg.Add(back);
	    }
	    else {  //right connections
	      gen_arg.Add(back);
              gen_arg.Add(front);
	    }

	  }
	  MapReduceInfo mr_info = new MapReduceInfo( (ISender) sender,
	 		                              new MapReduceArgs(this.TaskName,
					               	             mr_args.MapArg,
								     gen_arg,
								     mr_args.ReduceArg,
								     timeout // timeout
								     ));
          Log("{0}: {1}, adding address: {2} to sender list, range start: {3}, range end: {4}",
				    this.TaskName, _node.Address, next_c.Address,
				    gen_arg[0], gen_arg[1]);
	  last = next_addr;
	  retval.Add(mr_info);
	}
      }
      return retval;
    }    
    /**
     * Generates tree for bounded broadcast. Algorithm works as follows:
     * The goal is to broadcast to all nodes in range [local_address, end).
     * Given a range [local_address, b), determine all connections that belong to this range.
     * Let the connections be b_1, b_2, ..... b_n.
     * To connection bi assign the range [b_i, b_{i+1}).
     * To the connection bn assign range [b_n, end).]
     */
    public override void GenerateTree(Channel q, MapReduceArgs mr_args) 
    {
      object gen_arg = mr_args.GenArg;
      string end_range = gen_arg as string;
      Log("generating child tree, range end: {0}.", end_range);
      AHAddress end_addr = (AHAddress) AddressParser.Parse(end_range);
      AHAddress start_addr = _node.Address as AHAddress;
      //we are at the start node, here we go:
      ConnectionTable tab = _node.ConnectionTable;
      ConnectionList structs = tab.GetConnections(ConnectionType.Structured);
      ArrayList retval = new ArrayList();

      if (structs.Count > 0) {
        Connection curr_con = structs.GetLeftNeighborOf(_node.Address);
        int curr_idx = structs.IndexOf(curr_con.Address);
        //keep going until we leave the range
        int count = 0;
        ArrayList con_list = new ArrayList();
        while (count++ < structs.Count && ((AHAddress) curr_con.Address).IsBetweenFromLeft(start_addr, end_addr)) {
          con_list.Add(curr_con);
          //Log("adding connection: {0} to list.", curr_con.Address);
          curr_idx  = (curr_idx + 1)%structs.Count;
          curr_con = structs[curr_idx];
        }
        
        Log("{0}: {1}, number of child connections: {2}", 
            this.TaskName, _node.Address, con_list.Count);
        for (int i = 0; i < con_list.Count; i++) {
          MapReduceInfo mr_info = null;
          ISender sender = null;
          Connection con = (Connection) con_list[i];
          sender = (ISender) con.Edge;
          //check if last connection
          if (i == con_list.Count - 1) {
            mr_info = new MapReduceInfo( (ISender) sender, 
                                         new MapReduceArgs(this.TaskName, 
                                                           mr_args.MapArg, //map argument
                                                           end_range, //generate argument
                                                           mr_args.ReduceArg //reduce argument
                                                           ));
            
            Log("{0}: {1}, adding address: {2} to sender list, range end: {3}", 
                this.TaskName, _node.Address, 
                con.Address, end_range);
            retval.Add(mr_info);
          }
          else {
            string child_end = ((Connection) con_list[i+1]).Address.ToString();
            mr_info = new MapReduceInfo( sender,
                                         new MapReduceArgs(this.TaskName,
                                                           mr_args.MapArg, 
                                                           child_end,
                                                           mr_args.ReduceArg));
            Log("{0}: {1}, adding address: {2} to sender list, range end: {3}", 
                this.TaskName, _node.Address, 
                con.Address, child_end);
            retval.Add(mr_info);
          }
        }
      }
      q.Enqueue( retval.ToArray(typeof(MapReduceInfo)));
    }