private static void Crawl() { int count = 0, consistency = 0; NodeMapping nm = (NodeMapping) nodes.GetByIndex(0); Node lnode = nm.Node; Address rem_addr = lnode.Address, prev = null, first_left = null; bool failed = false; try { do { Console.WriteLine("Current address: " + rem_addr); ISender sender = new AHGreedySender(lnode, rem_addr); BlockingQueue q = new BlockingQueue(); lnode.Rpc.Invoke(sender, q, "sys:link.GetNeighbors"); RpcResult res = (RpcResult) q.Dequeue(); Hashtable ht = (Hashtable) res.Result; Address tmp = AddressParser.Parse((String) ht["left"]); Address next = AddressParser.Parse((String) ht["right"]); if(prev != null && tmp.Equals(prev)) { consistency++; } else { first_left = tmp; } if(next == lnode.Address && first_left == rem_addr) { consistency++; } prev = rem_addr; rem_addr = next; q.Close(); count++; } while((rem_addr != lnode.Address) && (count < nodes.Count)); } catch(Exception e) { failed = true; Console.WriteLine("Crawl failed due to exception..."); Console.WriteLine(e); } if(!failed) { if(count != nodes.Count) { Console.WriteLine("Crawl failed due to missing nodes!"); Console.WriteLine("Expected nodes: {0}, found: {1}.", nodes.Count, count); } else if(consistency != count) { Console.WriteLine("Crawl failed due to bad consistency!"); Console.WriteLine("Expected consistency: {0}, actual: {1}.", count, consistency); } else { Console.WriteLine("Crawl succeeded!"); } } }
public void Start() { foreach(NodeMapping nm_from in _nodes.Values) { foreach(NodeMapping nm_to in _nodes.Values) { if(nm_from == nm_to) { continue; } ISender sender = new AHGreedySender(nm_from.Node, nm_to.Node.Address); Channel q = new Channel(1); q.CloseEvent += Callback; try { nm_from.Node.Rpc.Invoke(sender, q, "sys:link.Ping", 0); _count++; _waiting_on++; } catch(Exception e) { Console.WriteLine(e); } } } }
protected void CrawlNext(Address addr) { bool finished = false; if(_log && _crawled.Count < _count) { Console.WriteLine("Current address: " + addr); } if(_crawled.Contains(addr)) { finished = true; } else { _crawled.Add(addr, true); try { ISender sender = new AHGreedySender(_node, addr); Channel q = new Channel(1); q.CloseEvent += CrawlHandler; _node.Rpc.Invoke(sender, q, "sys:link.GetNeighbors"); } catch(Exception e) { if(_log) { Console.WriteLine("Crawl failed" + e); } finished = true; } } if(finished) { Interlocked.Exchange(ref _done, 1); if(_log) { Console.WriteLine("Crawl stats: {0}/{1}", _crawled.Count, _count); Console.WriteLine("Consistency: {0}/{1}", _consistency, _crawled.Count); Console.WriteLine("Finished in: {0}", (DateTime.UtcNow - _start)); } } }
/** * When a node is out of the range, this method is called. * This method tries to find the nearest node to the middle of range using greedty algorithm. * return list of MapReduceInfo */ private ArrayList GenerateTreeOutRange(AHAddress start, AHAddress end, MapReduceArgs mr_args, int timeout) { ArrayList retval = new ArrayList(); BigInteger up = start.ToBigInteger(); BigInteger down = end.ToBigInteger(); BigInteger mid_range = (up + down) /2; if (mid_range % 2 == 1) {mid_range = mid_range -1; } AHAddress mid_addr = new AHAddress(mid_range); if (!mid_addr.IsBetweenFromLeft(start, end) ) { mid_range += Address.Half; mid_addr = new AHAddress(mid_range); } ArrayList gen_arg = new ArrayList(); if (NextGreedyClosest(mid_addr) != null ) { AHGreedySender ags = new AHGreedySender(_node, mid_addr); string start_range = start.ToString(); string end_range = end.ToString(); gen_arg.Add(start_range); gen_arg.Add(end_range); MapReduceInfo mr_info = new MapReduceInfo( (ISender) ags, new MapReduceArgs(this.TaskName, mr_args.MapArg, gen_arg, mr_args.ReduceArg, timeout)); Log("{0}: {1}, out of range, moving to the closest node to mid_range: {2} to target node, range start: {3}, range end: {4}", this.TaskName, _node.Address, mid_addr, start, end); retval.Add(mr_info); } else { // cannot find a node in the range. } return retval; }
/** * Callback function that is invoked when TargetSelector fetches candidate scores in a range. * Initiates connection setup. * Node: All connection messages can be tagged with a token string. This token string is currenly being * used to keep the following information about a shortcut: * 1. The node who initiated the shortcut setup. * 2. The random target near which shortcut was chosen. * @param start address pointing to the start of range to query. * @param score_table list of candidate addresses sorted by score. * @param current currently selected optimal (nullable) */ protected void CreateShortcutCallback(Address start, SortedList score_table, Address current) { if (score_table.Count > 0) { /** * we remember our address and the start of range inside the token. * token is the concatenation of * (a) local node address * (b) random target for the range queried by target selector */ string token = _node.Address + start.ToString(); //connect to the min_target Address min_target = (Address) score_table.GetByIndex(0); ISender send = null; if (start.Equals(min_target)) { //looks like the target selector simply returned our random address if (LogEnabled) { ProtocolLog.Write(ProtocolLog.SCO, String.Format("SCO local: {0}, Connecting (shortcut) to min_target: {1} (greedy), random_target: {2}.", _node.Address, min_target, start)); } //use a greedy sender send = new AHGreedySender(_node, min_target); } else { if (LogEnabled) { ProtocolLog.Write(ProtocolLog.SCO, String.Format("SCO local: {0}, Connecting (shortcut) to min_target: {1} (exact), random_target: {2}.", _node.Address, min_target, start)); } //use exact sender send = new AHExactSender(_node, min_target); } ConnectTo(send, min_target, STRUC_SHORT, token); } }
/// <summary>This is the generic Put that is used by both the regular Put /// and Create methods. The use of the unique variable differentiates the /// two. This is asynchronous. Results are stored in the Channel returns. /// Creates and Puts return true if successful or exception if there are /// network errors in adding the entry, creates also fail if a previous /// entry exists. The work of determining success is handled in /// PutEnqueueHandler and PutCloseHandler.</summary> /// <param name="key">The index to store the value at.</param> /// <param name="value">The value to store.</param> /// <param name="ttl">The dht lease time for the key:value pair.</param> /// <param name="returns">The Channel where the result will be placed.</param> /// <param name="unique">True to do a create, false otherwise.</param> public void AsyncPut(MemBlock key, MemBlock value, int ttl, Channel returns, bool unique) { if(!_online) { throw new DhtException("The Node is (going) offline, DHT is offline."); } AsDhtPutState adps = new AsDhtPutState(returns); MemBlock[] brunet_address_for_key = MapToRing(key); Channel[] q = new Channel[DEGREE]; lock(_adps_table.SyncRoot) { for (int k = 0; k < DEGREE; k++) { Channel queue = new Channel(1); queue.CloseEvent += this.PutCloseHandler; _adps_table[queue] = adps; q[k] = queue; } } for (int k = 0; k < DEGREE; k++) { Address target = new AHAddress(brunet_address_for_key[k]); AHSender s = new AHGreedySender(Node, target); _rpc.Invoke(s, q[k], "dht.Put", brunet_address_for_key[k], value, ttl, unique); } }
/// <summary>Restores any of the Dht results that don't return all their /// values. We only get here at the end of a Dht return operation.</summary> /// <remarks>This analyzes the holes and fills them in individually. This only /// fills holes where there was a positive result (MAJORITY of results /// received).</remarks> /// <param name="adgs">The AsDhtGetState to analyze for follow up.</param> protected void GetFollowUp(AsDhtGetState adgs) { foreach (DictionaryEntry de in adgs.results) { if(de.Value == null || de.Key == null) { continue; } Hashtable res = (Hashtable) de.Value; if(res.Count < MAJORITY || res.Count == DEGREE) { if(res.Count < MAJORITY) { if(Dht.DhtLog.Enabled) { ProtocolLog.Write(Dht.DhtLog, String.Format( "Failed get count:total = {0}:{1}", res.Count, DEGREE)); } } res.Clear(); continue; } MemBlock value = (MemBlock) de.Key; int ttl = (int) adgs.ttls[value] / res.Count; if(Dht.DhtLog.Enabled) { ProtocolLog.Write(Dht.DhtLog, String.Format( "Doing follow up put count:total = {0}:{1}", res.Count, DEGREE)); } for(int i = 0; i < DEGREE; i++) { if(!res.Contains(i)) { MemBlock key = adgs.brunet_address_for_key[i]; Channel queue = new Channel(); Address target = new AHAddress(key); AHSender s = new AHGreedySender(Node, target); try { _rpc.Invoke(s, queue, "dht.Put", key, value, ttl, false); } catch(Exception) {} } } res.Clear(); } adgs.ttls.Clear(); adgs.results.Clear(); }
/// <remarks>This starts the get process by sending dht.Get to all the remote /// end points that contain the key we're looking up. The next step is /// is when the results are placed in the channel and GetEnqueueHandler is /// called or GetCloseHandler is called. This means the get needs to be /// stateful, that information is stored in the _adgs_table.</remarks> public void AsyncGet(MemBlock key, Channel returns) { if(!_online) { throw new DhtException("The Node is (going) offline, DHT is offline."); } // create a GetState and map in our table map its queues to it // so when we get a GetHandler we know which state to load AsDhtGetState adgs = new AsDhtGetState(returns); Channel[] q = new Channel[DEGREE]; lock(_adgs_table.SyncRoot) { for (int k = 0; k < DEGREE; k++) { Channel queue = new Channel(1); _adgs_table[queue] = adgs; q[k] = queue; } } // Setting up our Channels for (int k = 0; k < DEGREE; k++) { Channel queue = q[k]; queue.EnqueueEvent += this.GetEnqueueHandler; queue.CloseEvent += this.GetCloseHandler; adgs.queueMapping[queue] = k; } // Sending off the request! adgs.brunet_address_for_key = MapToRing(key); for (int k = 0; k < DEGREE; k++) { Address target = new AHAddress(adgs.brunet_address_for_key[k]); AHSender s = new AHGreedySender(Node, target); // 1024 is in there for backwards compatibility _rpc.Invoke(s, q[k], "dht.Get", adgs.brunet_address_for_key[k], 1024, null); } }