private static void Crawl() {
int count = 0, consistency = 0;
NodeMapping nm = (NodeMapping) nodes.GetByIndex(0);
Node lnode = nm.Node;
Address rem_addr = lnode.Address, prev = null, first_left = null;
bool failed = false;
try {
do {
Console.WriteLine("Current address: " + rem_addr);
ISender sender = new AHGreedySender(lnode, rem_addr);
BlockingQueue q = new BlockingQueue();
lnode.Rpc.Invoke(sender, q, "sys:link.GetNeighbors");
RpcResult res = (RpcResult) q.Dequeue();
Hashtable ht = (Hashtable) res.Result;
Address tmp = AddressParser.Parse((String) ht["left"]);
Address next = AddressParser.Parse((String) ht["right"]);
if(prev != null && tmp.Equals(prev)) {
consistency++;
}
else {
first_left = tmp;
}
if(next == lnode.Address && first_left == rem_addr) {
consistency++;
}
prev = rem_addr;
rem_addr = next;
q.Close();
count++;
} while((rem_addr != lnode.Address) && (count < nodes.Count));
}
catch(Exception e) {
failed = true;
Console.WriteLine("Crawl failed due to exception...");
Console.WriteLine(e);
}
if(!failed) {
if(count != nodes.Count) {
Console.WriteLine("Crawl failed due to missing nodes!");
Console.WriteLine("Expected nodes: {0}, found: {1}.", nodes.Count, count);
}
else if(consistency != count) {
Console.WriteLine("Crawl failed due to bad consistency!");
Console.WriteLine("Expected consistency: {0}, actual: {1}.", count, consistency);
}
else {
Console.WriteLine("Crawl succeeded!");
}
}
}
public void Start() {
foreach(NodeMapping nm_from in _nodes.Values) {
foreach(NodeMapping nm_to in _nodes.Values) {
if(nm_from == nm_to) {
continue;
}
ISender sender = new AHGreedySender(nm_from.Node, nm_to.Node.Address);
Channel q = new Channel(1);
q.CloseEvent += Callback;
try {
nm_from.Node.Rpc.Invoke(sender, q, "sys:link.Ping", 0);
_count++;
_waiting_on++;
} catch(Exception e) {
Console.WriteLine(e);
}
}
}
}
protected void CrawlNext(Address addr) {
bool finished = false;
if(_log && _crawled.Count < _count) {
Console.WriteLine("Current address: " + addr);
}
if(_crawled.Contains(addr)) {
finished = true;
} else {
_crawled.Add(addr, true);
try {
ISender sender = new AHGreedySender(_node, addr);
Channel q = new Channel(1);
q.CloseEvent += CrawlHandler;
_node.Rpc.Invoke(sender, q, "sys:link.GetNeighbors");
} catch(Exception e) {
if(_log) {
Console.WriteLine("Crawl failed" + e);
}
finished = true;
}
}
if(finished) {
Interlocked.Exchange(ref _done, 1);
if(_log) {
Console.WriteLine("Crawl stats: {0}/{1}", _crawled.Count, _count);
Console.WriteLine("Consistency: {0}/{1}", _consistency, _crawled.Count);
Console.WriteLine("Finished in: {0}", (DateTime.UtcNow - _start));
}
}
}
/**
* When a node is out of the range, this method is called.
* This method tries to find the nearest node to the middle of range using greedty algorithm.
* return list of MapReduceInfo
*/
private ArrayList GenerateTreeOutRange(AHAddress start, AHAddress end, MapReduceArgs mr_args, int timeout) {
ArrayList retval = new ArrayList();
BigInteger up = start.ToBigInteger();
BigInteger down = end.ToBigInteger();
BigInteger mid_range = (up + down) /2;
if (mid_range % 2 == 1) {mid_range = mid_range -1; }
AHAddress mid_addr = new AHAddress(mid_range);
if (!mid_addr.IsBetweenFromLeft(start, end) ) {
mid_range += Address.Half;
mid_addr = new AHAddress(mid_range);
}
ArrayList gen_arg = new ArrayList();
if (NextGreedyClosest(mid_addr) != null ) {
AHGreedySender ags = new AHGreedySender(_node, mid_addr);
string start_range = start.ToString();
string end_range = end.ToString();
gen_arg.Add(start_range);
gen_arg.Add(end_range);
MapReduceInfo mr_info = new MapReduceInfo( (ISender) ags,
new MapReduceArgs(this.TaskName,
mr_args.MapArg,
gen_arg,
mr_args.ReduceArg,
timeout));
Log("{0}: {1}, out of range, moving to the closest node to mid_range: {2} to target node, range start: {3}, range end: {4}",
this.TaskName, _node.Address, mid_addr, start, end);
retval.Add(mr_info);
}
else {
// cannot find a node in the range.
}
return retval;
}
/**
* Callback function that is invoked when TargetSelector fetches candidate scores in a range.
* Initiates connection setup.
* Node: All connection messages can be tagged with a token string. This token string is currenly being
* used to keep the following information about a shortcut:
* 1. The node who initiated the shortcut setup.
* 2. The random target near which shortcut was chosen.
* @param start address pointing to the start of range to query.
* @param score_table list of candidate addresses sorted by score.
* @param current currently selected optimal (nullable)
*/
protected void CreateShortcutCallback(Address start, SortedList score_table, Address current) {
if (score_table.Count > 0) {
/**
* we remember our address and the start of range inside the token.
* token is the concatenation of
* (a) local node address
* (b) random target for the range queried by target selector
*/
string token = _node.Address + start.ToString();
//connect to the min_target
Address min_target = (Address) score_table.GetByIndex(0);
ISender send = null;
if (start.Equals(min_target)) {
//looks like the target selector simply returned our random address
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.SCO,
String.Format("SCO local: {0}, Connecting (shortcut) to min_target: {1} (greedy), random_target: {2}.",
_node.Address, min_target, start));
}
//use a greedy sender
send = new AHGreedySender(_node, min_target);
} else {
if (LogEnabled) {
ProtocolLog.Write(ProtocolLog.SCO,
String.Format("SCO local: {0}, Connecting (shortcut) to min_target: {1} (exact), random_target: {2}.",
_node.Address, min_target, start));
}
//use exact sender
send = new AHExactSender(_node, min_target);
}
ConnectTo(send, min_target, STRUC_SHORT, token);
}
}
/// <summary>This is the generic Put that is used by both the regular Put
/// and Create methods. The use of the unique variable differentiates the
/// two. This is asynchronous. Results are stored in the Channel returns.
/// Creates and Puts return true if successful or exception if there are
/// network errors in adding the entry, creates also fail if a previous
/// entry exists. The work of determining success is handled in
/// PutEnqueueHandler and PutCloseHandler.</summary>
/// <param name="key">The index to store the value at.</param>
/// <param name="value">The value to store.</param>
/// <param name="ttl">The dht lease time for the key:value pair.</param>
/// <param name="returns">The Channel where the result will be placed.</param>
/// <param name="unique">True to do a create, false otherwise.</param>
public void AsyncPut(MemBlock key, MemBlock value, int ttl, Channel returns, bool unique) {
if(!_online) {
throw new DhtException("The Node is (going) offline, DHT is offline.");
}
AsDhtPutState adps = new AsDhtPutState(returns);
MemBlock[] brunet_address_for_key = MapToRing(key);
Channel[] q = new Channel[DEGREE];
lock(_adps_table.SyncRoot) {
for (int k = 0; k < DEGREE; k++) {
Channel queue = new Channel(1);
queue.CloseEvent += this.PutCloseHandler;
_adps_table[queue] = adps;
q[k] = queue;
}
}
for (int k = 0; k < DEGREE; k++) {
Address target = new AHAddress(brunet_address_for_key[k]);
AHSender s = new AHGreedySender(Node, target);
_rpc.Invoke(s, q[k], "dht.Put", brunet_address_for_key[k], value, ttl, unique);
}
}
/// <summary>Restores any of the Dht results that don't return all their
/// values. We only get here at the end of a Dht return operation.</summary>
/// <remarks>This analyzes the holes and fills them in individually. This only
/// fills holes where there was a positive result (MAJORITY of results
/// received).</remarks>
/// <param name="adgs">The AsDhtGetState to analyze for follow up.</param>
protected void GetFollowUp(AsDhtGetState adgs) {
foreach (DictionaryEntry de in adgs.results) {
if(de.Value == null || de.Key == null) {
continue;
}
Hashtable res = (Hashtable) de.Value;
if(res.Count < MAJORITY || res.Count == DEGREE) {
if(res.Count < MAJORITY) {
if(Dht.DhtLog.Enabled) {
ProtocolLog.Write(Dht.DhtLog, String.Format(
"Failed get count:total = {0}:{1}", res.Count, DEGREE));
}
}
res.Clear();
continue;
}
MemBlock value = (MemBlock) de.Key;
int ttl = (int) adgs.ttls[value] / res.Count;
if(Dht.DhtLog.Enabled) {
ProtocolLog.Write(Dht.DhtLog, String.Format(
"Doing follow up put count:total = {0}:{1}", res.Count, DEGREE));
}
for(int i = 0; i < DEGREE; i++) {
if(!res.Contains(i)) {
MemBlock key = adgs.brunet_address_for_key[i];
Channel queue = new Channel();
Address target = new AHAddress(key);
AHSender s = new AHGreedySender(Node, target);
try {
_rpc.Invoke(s, queue, "dht.Put", key, value, ttl, false);
}
catch(Exception) {}
}
}
res.Clear();
}
adgs.ttls.Clear();
adgs.results.Clear();
}
/// <remarks>This starts the get process by sending dht.Get to all the remote
/// end points that contain the key we're looking up. The next step is
/// is when the results are placed in the channel and GetEnqueueHandler is
/// called or GetCloseHandler is called. This means the get needs to be
/// stateful, that information is stored in the _adgs_table.</remarks>
public void AsyncGet(MemBlock key, Channel returns) {
if(!_online) {
throw new DhtException("The Node is (going) offline, DHT is offline.");
}
// create a GetState and map in our table map its queues to it
// so when we get a GetHandler we know which state to load
AsDhtGetState adgs = new AsDhtGetState(returns);
Channel[] q = new Channel[DEGREE];
lock(_adgs_table.SyncRoot) {
for (int k = 0; k < DEGREE; k++) {
Channel queue = new Channel(1);
_adgs_table[queue] = adgs;
q[k] = queue;
}
}
// Setting up our Channels
for (int k = 0; k < DEGREE; k++) {
Channel queue = q[k];
queue.EnqueueEvent += this.GetEnqueueHandler;
queue.CloseEvent += this.GetCloseHandler;
adgs.queueMapping[queue] = k;
}
// Sending off the request!
adgs.brunet_address_for_key = MapToRing(key);
for (int k = 0; k < DEGREE; k++) {
Address target = new AHAddress(adgs.brunet_address_for_key[k]);
AHSender s = new AHGreedySender(Node, target);
// 1024 is in there for backwards compatibility
_rpc.Invoke(s, q[k], "dht.Get", adgs.brunet_address_for_key[k], 1024, null);
}
}
