/// <summary> /// Adds a peer to the routing table /// </summary> /// <returns><c>true</c> if the item was added; <c>false</c> otherwise.</returns> /// <param name="peer">Peer.</param> /// <param name="isNew">A flag indicating if the item was new</param> public bool Add(PeerInfo peer, out bool isNew) { if (peer == null) { throw new ArgumentNullException(nameof(peer)); } isNew = false; // Get the closest leaf node var node = GetClosestLeaf(peer.Key); // Check if we are re-inserting a known key for (var i = 0; i < node.Items.Count; i++) { if (node.Items[i].Key.Equals(peer.Key)) { // Are we simply re-freshing the peer? if (node.Items[i].Address.Equals(peer.Address)) { // Refreshing, move the item to the top node.Items.RemoveAt(i); node.Items.Add(peer); return(true); } // We are not replacing keys return(false); } } // Keep splitting until we have space while (true) { // Do we have space? if (node.Items.Count == m_k) { // No space, but should we make space? if (PrefixMatches(node, peer.Key) || node == m_root) { SplitNode(node); node = PrefixMatches(node, peer.Key) ? node.Right : node.Left; continue; } // No more space return(false); } // We have space, so just add it node.Items.Add(peer); m_count++; isNew = true; return(true); } }
/// <summary> /// Listens to the socket and registers all new requests /// </summary> /// <returns>The async.</returns> /// <param name="selfinfo">Selfinfo.</param> /// <param name="sock">Sock.</param> private static Task ListenAsync(PeerInfo selfinfo, TcpListener sock) { // Set up a channel for sending sockets var sockchan = Channel.Create <TcpClient>(); var listener = Task.Run(async() => { while (true) { await sockchan.WriteAsync(await sock.AcceptTcpClientAsync()); } }); // Handle each request var handlers = Skeletons.CollectAsync(sockchan, async client => { var peer = PeerConnection.CreatePeer(selfinfo, selfinfo, client.GetStream()); // Ping the peer var res = await peer.Item2.SendConnectionRequestAsync(null, null, new Protocol.Request() { Operation = Protocol.Operation.Ping, Self = selfinfo, }); // Register this peer with the broker await Channels.ConnectionBrokerRegistrations.Get().WriteAsync( new ConnectionRegistrationRequest() { IsTerminate = false, UpdateRouting = true, Peer = res.Response.Self, Channel = peer.Item2 } ); // If we get anything back, register with the routing table if (res.Response.Peers != null) { var router = Channels.RoutingTableRequests.Get(); log.Debug($"Ping response had {res.Response.Peers.Count} peers, registering with routing table"); foreach (var np in res.Response.Peers) { await router.AddPeerAsync(np.Key, np); } } log.Debug("Completed initial ping sequence"); }, 10); return(Task.WhenAll(listener, handlers)); }
/// <summary> /// Handles a remote ping request /// </summary> /// <returns>An awaitable task.</returns> /// <param name="self">The node information.</param> /// <param name="req">The request to handle.</param> private static Task HandlePingRequestAsync(PeerInfo self, ConnectionRequest req) { log.Debug($"Converting ping request to node lookup ({req.RequestID})"); return(HandleNodeLookupRequestAsync(self, new ConnectionRequest() { Key = self.Key, EndPoint = self.Address, RequestID = req.RequestID, Response = req.Response, Request = new Protocol.Request() { Target = self.Key, Operation = Protocol.Operation.Ping } })); }
/// <summary> /// Handles a refresh operation by requesting routing data from peers /// </summary> /// <returns>The refresh operation.</returns> /// <param name="selfinfo">The nodes own information.</param> /// <param name="request">Request.</param> /// <param name="k">The number of peers to request</param> public static async Task HandleRefreshOperation(PeerInfo selfinfo, PeerRequest request, int k) { var res = await VisitClosestPeers(selfinfo, request.Key ?? selfinfo.Key, request.Key == null?k : 1, 1, new Protocol.Request() { Self = selfinfo, Operation = Protocol.Operation.FindPeer, Target = selfinfo.Key }); if (request.Response != null) { await request.Response.WriteAsync(new PeerResponse() { SuccessCount = res.Count }); } }
/// <summary> /// Handles a request to add a value to the DHT /// </summary> /// <returns>The add operation.</returns> /// <param name="selfinfo">Selfinfo.</param> /// <param name="request">The request to handle.</param> /// <param name="k">The number of copies to store</param> public static async Task HandleAddOperation(PeerInfo selfinfo, PeerRequest request, int k) { var key = Key.ComputeKey(request.Data); log.Debug($"Handling the add request"); var stored = await VisitClosestPeers(selfinfo, key, k, k, new Protocol.Request() { Self = selfinfo, Operation = Protocol.Operation.Store, Data = request.Data, Target = key }); await request.Response.WriteAsync(new PeerResponse() { SuccessCount = stored.Count }); }
/// <summary> /// Handles a remote store request /// </summary> /// <returns>An awaitable task.</returns> /// <param name="self">The node information.</param> /// <param name="request">The request to handle.</param> private static async Task HandleStoreRequestAsync(PeerInfo self, ConnectionRequest request) { log.Debug($"Query MRU ({request.RequestID})"); var res = await Channels.MRURequests.Get() .SendAddAsync(request.Request.Target, request.Request.Data); log.Debug($"Got response, forwarding to requester ({request.RequestID}) ..."); await request.Response.WriteAsync(new ConnectionResponse() { RequestID = request.RequestID, Response = new Protocol.Response() { Self = self, Success = res, } }); log.Debug($"Completed store query ({request.RequestID})"); }
/// <summary> /// Runs the discovery process /// </summary> /// <returns>An awaitable task.</returns> /// <param name="self">The peer making the query.</param> /// <param name="endPoints">The end points to query.</param> public static Task RunAsync(PeerInfo self, EndPoint[] endPoints) { if (self == null) { throw new ArgumentNullException(nameof(self)); } if (endPoints == null) { throw new ArgumentNullException(nameof(endPoints)); } var targets = endPoints.Where(x => !self.Address.Equals(x)); log.Debug($"Performing discovery on {targets.Count()} peer"); return(TaskPool.RunParallelAsync( targets, x => QueryPeerAsync(self, x), (x, ex) => log.Warn($"Discovery failed for peer {x}", ex) ).ContinueWith(x => log.Debug("Discovery process completed"))); }
/// <summary> /// Broadcasts the store operation to the peers. /// </summary> /// <returns>An awaitable task.</returns> /// <param name="request">The internal request to handle.</param> private static async Task BroadcastValueAsync(PeerInfo self, MRUInternalStore request) { log.Debug($"Broadcasting store to {request.Peers.Count - 1} peers"); var tp = new TaskPool <PeerInfo>(5, (x, ex) => log.Warn($"Failed to broadcast to peer {x.Key} - {x.Address}", ex)); var cb = Channels.ConnectionBrokerRequests.Get(); foreach (var n in request.Peers.Where(x => !self.Key.Equals(x.Key))) { // Send a store request to all the k nearest peers await tp.Run(n, peer => cb.SendConnectionRequestAsync(peer.Address, peer.Key, new Protocol.Request() { Operation = Protocol.Operation.Store, Data = request.Data, Target = request.Key, Self = self } )); } }
/// <summary> /// Handles the find operation /// </summary> /// <returns>An awaitable task.</returns> /// <param name="selfinfo">The nodes own information.</param> /// <param name="request">The request to handle.</param> /// <param name="k">The redundancy count</param> public static async Task HandleFindOperation(PeerInfo selfinfo, PeerRequest request, int k) { // Query the local value store first var data = await Channels.MRURequests.Get().SendGetAsync(request.Key); if (data != null) { log.Debug($"Local lookup for key succeeded"); await request.Response.WriteAsync(new PeerResponse() { Data = data, SuccessCount = 1 }); return; } var res = await VisitClosestPeers(selfinfo, request.Key, k, 1, new Protocol.Request() { Self = selfinfo, Operation = Protocol.Operation.FindValue, Target = request.Key }); var result = res.Where(x => x.Success).FirstOrDefault(); if (result.Success) { log.Debug($"Lookup succeeded for key, (re-)adding to local table"); await Channels.MRURequests.Get().SendAddAsync(request.Key, result.Data); } await request.Response.WriteAsync(new PeerResponse() { Data = result.Data, SuccessCount = res.Count }); }
/// <summary> /// Queries a single peer, trying to obtain the peer information and updates the routing table /// </summary> /// <returns>An awaitable task.</returns> /// <param name="self">The peer making the query.</param> /// <param name="endPoint">The end point to query.</param> private static async Task QueryPeerAsync(PeerInfo self, EndPoint endPoint) { if (self == null) { throw new ArgumentNullException(nameof(self)); } if (endPoint == null) { throw new ArgumentNullException(nameof(endPoint)); } if (!(endPoint is IPEndPoint)) { throw new ArgumentException($"Can only connect to an {nameof(IPEndPoint)}", nameof(endPoint)); } log.Debug($"Performing discovery on {endPoint} ..."); var response = await Channels.ConnectionBrokerRequests.Get().SendConnectionRequestAsync(endPoint, null, new Protocol.Request() { Self = self, Operation = Protocol.Operation.FindPeer, Target = self.Key }); log.Debug($"Discovery response arrived from {endPoint} ..."); if (response.Exception != null) { log.DebugFormat("Failed to contact peer {0}: {1}", endPoint, response.Exception); return; } log.Debug($"Discovery complete for {endPoint}"); }
/// <summary> /// Creates a new peer for the given endpoint /// </summary> /// <returns>A task for the peer, and a communication channel.</returns> /// <param name="self">This peer's information</param> /// <param name="remote">The remote peer's information</param> /// <param name="connecthandler">The method used to obtain the stream</param> /// <param name="maxparallel">The maximum number of requests to handle in parallel</param> public static Tuple <Task, IWriteChannel <ConnectionRequest> > CreatePeer(PeerInfo self, PeerInfo remote, Func <Task <Stream> > connecthandler, int maxparallel = REQ_BUFFER_SIZE) { if (self == null) { throw new ArgumentNullException(nameof(self)); } if (connecthandler == null) { throw new ArgumentNullException(nameof(connecthandler)); } var source = Channel.Create <ConnectionRequest>(); var sink = Channel.Create <ConnectionRequest>(); return(new Tuple <Task, IWriteChannel <ConnectionRequest> >( Task.WhenAll( Task.Run(() => RunSingleConnection(self, remote, connecthandler, sink.AsRead(), maxparallel)), TaskPool.RunParallelAsync( source.AsRead(), x => sink.WriteAsync(x), REQ_BUFFER_SIZE, async(c, x) => { try { await c.Response.WriteAsync(new ConnectionResponse() { Exception = x }); } catch (Exception ex) { log.Warn("Failed to send error response", ex); } } ) ), source.AsWrite() )); }
/// <summary> /// Creates a new peer for the given endpoint /// </summary> /// <returns>A task for the peer, and a communication channel.</returns> /// <param name="self">This peer's information</param> /// <param name="remote">The remote peer's information</param> /// <param name="stream">The stream to use.</param> /// <param name="maxparallel">The maximum number of requests to handle in parallel</param> public static Tuple <Task, IWriteChannel <ConnectionRequest> > CreatePeer(PeerInfo self, PeerInfo remote, Stream stream, int maxparallel = REQ_BUFFER_SIZE) { return(CreatePeer(self, remote, () => Task.FromResult(stream), maxparallel)); }
/// <summary> /// Runs the broker process. /// </summary> /// <param name="node">This nodes information</param> /// <param name="maxconnections">The maximum number of connections to allow</param> /// <returns>An awaitable task.</returns> public static Task RunAsync(PeerInfo node, int maxconnections = 50) { // The primary table for finding peers var peers = new Dictionary <EndPoint, Tuple <Task, IWriteChannel <ConnectionRequest> > >(); // The peers listed by key var peersbykey = new Dictionary <Key, EndPoint>(); // The MRU cache of peers var mrucache = new MRUCache <EndPoint, Key>(maxconnections, TimeSpan.FromDays(10)); return(AutomationExtensions.RunTask( new { Request = Channels.ConnectionBrokerRequests.ForRead, Registrations = Channels.ConnectionBrokerRegistrations.ForRead, Stats = Channels.ConnectionBrokerStats.ForRead, SelfHandler = Channels.RemoteRequests.ForWrite, Routing = Channels.RoutingTableRequests.ForWrite }, async self => { log.Debug($"Broker is now running"); while (true) { log.Debug($"Broker is waiting for requests ..."); var mreq = await MultiChannelAccess.ReadFromAnyAsync( self.Stats.RequestRead(), self.Registrations.RequestRead(), self.Request.RequestRead() ); if (mreq.Channel == self.Stats) { log.Debug($"Broker got stat request"); var req = (IWriteChannel <ConnectionStatsResponse>)mreq.Value; await req.WriteAsync(new ConnectionStatsResponse() { EndPoints = peers.Count, Keys = peersbykey.Count, Stats = (Channels.ConnectionBrokerRequests.Get() as ProfilingChannel <ConnectionRequest>)?.ReportStats() }); } else if (mreq.Channel == self.Registrations) { var req = (ConnectionRegistrationRequest)mreq.Value; log.Debug($"Broker got {(req.IsTerminate ? "termination" : "registration")} request"); if (req.IsTerminate) { // Make sure we do not have stale stuff in the MRU cache if (req.Peer != null && req.Peer.Address != null) { mrucache.Remove(req.Peer.Address); } if (req.Peer.Address != null && peers.TryGetValue(req.Peer.Address, out var c) && c.Item2 == req.Channel) { peers.Remove(req.Peer.Address); if (req.Peer.Key != null) { peersbykey.Remove(req.Peer.Key); } } if (req.UpdateRouting) { log.Debug($"Removing peer in routing table due to termination of connection {req.Peer.Key} - {req.Peer.Address}"); await self.Routing.RemovePeerAsync(req.Peer.Key); } } else { if (req.Peer.Address != null && peers.TryGetValue(req.Peer.Address, out var c) && (c.Item2 == req.Channel || c == null)) { if (c == null) { peers[req.Peer.Address] = new Tuple <Task, IWriteChannel <ConnectionRequest> >(null, req.Channel); } if (!peersbykey.ContainsKey(req.Peer.Key)) { peersbykey[req.Peer.Key] = req.Peer.Address; } } if (req.UpdateRouting) { log.Debug($"Adding new peer to routing table {req.Peer.Key} - {req.Peer.Address}"); await self.Routing.AddPeerAsync(req.Peer.Key, req.Peer); } } } else { var req = (ConnectionRequest)mreq.Value; log.Debug($"Broker got connection request for {req.EndPoint}"); // Check if we request ourselves if (node.Key.Equals(req.Key) || node.Address.Equals(req.EndPoint)) { log.Debug($"Broker got self-request, forwarding to owner"); await self.SelfHandler.WriteAsync(req); continue; } Tuple <Task, IWriteChannel <ConnectionRequest> > peer = null; try { // Existing connection, update MRU var overflow = mrucache.Add(req.EndPoint, req.Key); // If we have too many connections, kill one now if (overflow != null) { // We could make this also take the closest k peers into account log.Debug($"Broker has too many connections, closing {req.EndPoint}"); await peers[overflow].Item2.RetireAsync(); } if (!peers.TryGetValue(req.EndPoint, out peer)) { log.Debug($"Broker is starting a connection to {req.EndPoint}"); mrucache.Add(req.EndPoint, req.Key); peer = peers[req.EndPoint] = PeerConnection.CreatePeer( node, new PeerInfo(req.Key, req.EndPoint), () => ConnectToPeerAsync(req.EndPoint), REQ_BUFFER_SIZE ); if (req.Key != null) { peersbykey[req.Key] = req.EndPoint; } } await peer.Item2.WriteAsync(req); } catch (Exception ex) { log.Warn("Failed to send request to peer", ex); try { await req.Response.WriteAsync(new ConnectionResponse() { Exception = ex }); } catch (Exception ex2) { log.Warn("Failed to write failure response", ex2); } if (peer != null) { try { peer.Item2.AsWriteOnly().Dispose(); } catch (Exception ex2) { log.Warn("Failed to terminate write channel", ex2); } try { await peer.Item1; } catch (Exception ex2) { log.Warn("Peer connection stopped with error", ex2); } } peers.Remove(req.EndPoint); } } } } )); }
/// <summary> /// Runs a process that handles remote requests /// </summary> /// <param name="selfinfo">Description of the owning node</param> /// <param name="maxparallel">The maximum number of requests to handle in parallel</param> /// <returns>The async.</returns> public static Task RunAsync(PeerInfo selfinfo, int maxparallel = 10) { return(AutomationExtensions.RunTask(new { Requests = Channels.RemoteRequests.ForRead, Routing = Channels.RoutingTableRequests.ForWrite }, async self => { log.Debug("Running the remote handler"); // Set up an error handler Func <ConnectionRequest, Exception, Task> errorHandler = async(t, ex) => { try { await t.Response.WriteAsync(new ConnectionResponse() { Exception = ex, RequestID = t.RequestID }); } catch (Exception ex2) { log.Warn("Failed to send error response", ex2); } }; using (var tp = new TaskPool <ConnectionRequest>(maxparallel, errorHandler)) while (true) { log.Debug("Remote handler is waiting for requests ..."); var req = await self.Requests.ReadAsync(); log.Debug($"Remote handler got a {req.Request.Operation} request ({req.RequestID})"); await tp.Run(req, async() => { log.Debug($"Remote handler is processing {req.Request.Operation} request ({req.RequestID})"); if (req.Key != null && req.EndPoint != null) { log.Debug($"Updating route table with remote request data {req.Key} - {req.EndPoint}"); await self.Routing.AddPeerAsync(req.Key, new PeerInfo(req.Key, req.EndPoint)); } switch (req.Request.Operation) { case Protocol.Operation.Ping: await HandlePingRequestAsync(selfinfo, req); break; case Protocol.Operation.Store: await HandleStoreRequestAsync(selfinfo, req); break; case Protocol.Operation.FindValue: await HandleFindRequestAsync(selfinfo, req); break; case Protocol.Operation.FindPeer: await HandleNodeLookupRequestAsync(selfinfo, req); break; default: await req.Response.WriteAsync(new ConnectionResponse() { RequestID = req.RequestID, Exception = new Exception($"Invalid operation: {req.Request.Operation}") }); break; } log.Debug($"Remote handler finished processing {req.Request.Operation} request ({req.RequestID})"); }); } })); }
/// <summary> /// Gets the <paramref name="count"/> closest live nodes to the <paramref name="key"/>. /// </summary> /// <returns>The closest live peers.</returns> /// <param name="selfinfo">Selfinfo.</param> /// <param name="key">The key to search for.</param> public static Task <List <PeerInfo> > GetClosestPeers(PeerInfo selfinfo, Key key, int count) { return(Channels.RoutingTableRequests.Get().LookupPeerAsync(key)); }
/// <summary> /// Visits the <paramref name="k"/> closest peers and sends the <paramref name="request"/> message to them /// </summary> /// <returns>The number of nodes visited.</returns> /// <param name="selfinfo">Selfinfo.</param> /// <param name="key">The key to use for findng the closest nodes.</param> /// <param name="k">The redundancy parameter.</param> /// <param name="succes_count">The number of success responses to find</param> /// <param name="request">The request to send.</param> private static async Task <List <Protocol.Response> > VisitClosestPeers(PeerInfo selfinfo, Key key, int k, int succes_count, Protocol.Request request) { KeyDistance closesttried = null; var peers = await GetClosestPeers(selfinfo, key, k); log.Debug($"Initial query gave {peers.Count} peers"); var used = new HashSet <Key>(); var lck = new object(); var closest = new List <PeerInfo>(); var cb = Channels.ConnectionBrokerRequests.Get(); var success = new List <Protocol.Response>(); using (var tp = new TaskPool <PeerInfo>(2, (p, ex) => log.Warn($"Request to peer {p.Key} - {p.Address} failed", ex))) while (success.Count < succes_count && (peers.Count + closest.Count > 0)) { peers = closest .Union(peers) // Remove dead items .Where(x => !used.Contains(x.Key)) // Always move closer to the target for find .Where(x => closesttried == null || (new KeyDistance(key, x.Key).CompareTo(closesttried)) <= 0) // Sort by distance .OrderBy(x => new KeyDistance(key, x.Key)) .ToList(); // Clean the list closest.Clear(); log.Debug($"Sending request to {peers.Count} peers"); // Get the peers var alloperations = peers.Select(x => tp.Run(x, async() => { log.Debug($"Success count {success.Count}, target: {succes_count}"); if (success.Count >= succes_count) { return; } log.Debug("Sending request via broker"); var r = cb.SendConnectionRequestAsync(x.Address, x.Key, request); // Don't try this again lock (lck) used.Add(x.Key); log.Debug($"Waiting for broker response {x.Key}..."); var res = await r; log.Debug($"Peer response for {x.Key} obtained ({(res.Exception != null ? "exception": (res.Response.Success ? "success" : "no hit"))})"); // Skip error nodes if (res.Exception != null) { log.Warn("Node request failed", res.Exception); //await Channels.ConnectionBrokerRegistrations.Get().WriteAsync(new ConnectionRegistrationRequest() { // IsTerminate = true, // UpdateRouting = true, // Peer = x //}); return; } // Record success if (res.Response.Success) { lock (lck) success.Add(res.Response); } // Stock up on peers, if any if (res.Response.Peers != null) { lock (lck) closest.AddRange(res.Response.Peers); } // If we are doing a find, narrow the scope if (request.Operation == Protocol.Operation.FindValue) { if (closesttried == null || new KeyDistance(key, x.Key).CompareTo(closesttried) < 0) { closesttried = new KeyDistance(key, x.Key); } } })); await Task.WhenAll(alloperations); await tp.FinishedAsync(); log.Debug($"Got {closest.Count} potential new peers"); if (closest.Count == 0) { break; } } return(success); }
/// <summary> /// Runs the router and forwarder /// </summary> /// <returns>An awaitable result.</returns> /// <param name="owner">The owner of the routing table.</param> /// <param name="k">The redundancy parameter.</param> /// <param name="buffersize">The size of the forwarding buffer.</param> public static Task RunAsync(PeerInfo owner, int k, int buffersize = 10) { return(AutomationExtensions.RunTask( new { Requests = Channels.RoutingTableRequests.ForRead, Stats = Channels.RoutingTableStats.ForRead, PeerReq = Channels.PeerRequests.ForWrite }, async self => { // Setup the routing table, and add us to it var table = new RoutingTable(owner.Key, k); table.Add(owner); log.Debug($"Router is now running"); // Set up the error handler Func <RoutingRequest, Exception, Task> errorHandler = async(t, ex) => { log.Warn("Routing operation failed, sending failure response to requester", ex); try { await t.Response.WriteAsync(new RoutingResponse() { Exception = ex, Succes = false }); } catch (Exception ex2) { log.Warn("Failed to forward error message", ex2); } }; using (var tp = new TaskPool <RoutingRequest>(buffersize, errorHandler)) while (true) { // Wait for requests log.Debug($"Router is waiting for requests ..."); var r = await MultiChannelAccess.ReadFromAnyAsync(self.Stats.RequestRead(), self.Requests.RequestRead()); if (r.Channel == self.Stats) { log.Debug($"Router got stat request"); var m = (IWriteChannel <RoutingStatsResponse>)r.Value; await m.WriteAsync(new RoutingStatsResponse() { Count = table.Count, Stats = (Channels.RoutingTableRequests.Get() as ProfilingChannel <RoutingRequest>)?.ReportStats() }); continue; } var data = (RoutingRequest)r.Value; try { log.Debug($"Router got request {data.Operation}"); // Multiplex the operation switch (data.Operation) { case RoutingOperation.Add: { var success = table.Add(data.Data, out var isNew); if (data.Response != null) { await tp.Run(data, () => data.Response.WriteAsync(new RoutingResponse() { Succes = true, IsNew = isNew })); } // If the peer is new, discover what peers it knows if (isNew) { log.Debug($"New peer, requesting refresh"); await tp.Run(data, () => self.PeerReq.WriteAsync(new PeerRequest() { Operation = PeerOperation.Refresh, Key = data.Data.Key })); log.Debug($"Peer refresh requested"); } break; } case RoutingOperation.Remove: { var sucess = table.RemoveKey(data.Key); if (data.Response != null) { await tp.Run(data, () => data.Response.WriteAsync(new RoutingResponse() { Succes = sucess })); } break; } case RoutingOperation.Lookup: { var peers = table.Nearest(data.Key, k, data.OnlyKBucket); await tp.Run(data, () => data.Response.WriteAsync(new RoutingResponse() { Succes = true, Peers = peers })); break; } default: throw new Exception($"Operation not supported: {data.Operation}"); } } catch (Exception ex) { await errorHandler(data, ex); } } })); }
/// <summary> /// Adds a peer to the routing table /// </summary> /// <returns><c>true</c> if the peer is new, <c>false</c> otherwise.</returns> /// <param name="channel">The channel to send the request on.</param> /// <param name="key">The key for the peer.</param> /// <param name="peer">The peer.</param> public static async Task <bool> AddPeerAsync(this IWriteChannel <RoutingRequest> channel, Key key, PeerInfo peer) { return((await SendMessageAsync <RoutingRequest, RoutingResponse>(channel, new RoutingRequest() { Operation = RoutingOperation.Add, Key = key, Data = peer })).IsNew); }
/// <summary> /// Runs a single peer connection, using the IPC link /// </summary> /// <returns>An awaitable task.</returns> /// <param name="self">This peer's information</param> /// <param name="remote">The remote peer's information</param> /// <param name="connecthandler">The method used to obtain the connection.</param> /// <param name="input">The channel for reading requests.</param> /// <param name="maxparallel">The maximum number of parallel handlers</param> private static async Task RunSingleConnection(PeerInfo self, PeerInfo remote, Func <Task <Stream> > connecthandler, IReadChannel <ConnectionRequest> input, int maxparallel) { // Get the local handler for remote requests var remotehandler = Channels.RemoteRequests.Get(); LeanIPC.IPCPeer connection = null; try { if (connecthandler == null) { throw new ArgumentNullException(nameof(connecthandler)); } if (input == null) { throw new ArgumentNullException(nameof(input)); } log.Debug($"Setting up connection to {remote?.Key}"); // Connect to the remote peer connection = new LeanIPC.IPCPeer(await connecthandler()); // Setup a handler for remote requests, that forwards responses from the remote handler connection.AddUserTypeHandler <Protocol.Request>( async(id, req) => { await connection.SendResponseAsync(id, (await remotehandler.SendConnectionRequestAsync(null, null, id, req)).Response); return(true); } ); var mainTask = connection.RunMainLoopAsync(self != remote); Key targetKey = null; // Grab a connection to update the routing table automatically var routingrequests = Channels.RoutingTableRequests.Get(); log.Debug($"Peer connection running {self.Key}, {self.Address}"); using (var tp = new TaskPool <ConnectionRequest>(maxparallel, (t, ex) => log.Warn("Unexpected error handling request", ex))) while (true) { log.Debug($"Peer connection is waiting for request ..."); // Get either a local or a remote request var req = await input.ReadAsync(); log.Debug($"Peer connection got request, handling on taskpool ..."); await tp.Run(req, () => { log.Debug($"Peer connection is forwarding a local {req.Request.Operation} request to the remote"); return(Task.Run(async() => { ConnectionResponse res; try { var p = await connection.SendAndWaitAsync <Protocol.Request, Protocol.Response>(req.Request); if (targetKey == null) { // Record the target key targetKey = p.Self.Key; if (remote == null || remote.Key == null) { remote = new PeerInfo(p.Self.Key, remote.Address); } // Write a registration request to the broker await Channels.ConnectionBrokerRegistrations.Get().WriteAsync( new ConnectionRegistrationRequest() { IsTerminate = false, UpdateRouting = true, Peer = remote } ); log.Debug($"Registering peer in routing table: {remote.Key} {remote.Address} ..."); await routingrequests.AddPeerAsync(remote.Key, remote); } if (p.Peers != null) { log.Debug($"Registering {p.Peers.Count} peers with the routing table ..."); foreach (var peer in p.Peers) { await routingrequests.AddPeerAsync(peer.Key, peer); } log.Debug($"Registered {p.Peers.Count} peers with the routing table"); } res = new ConnectionResponse() { Key = p.Self.Key, Response = p }; } catch (Exception ex) { log.Warn($"Failed to get result, sending error response", ex); res = new ConnectionResponse() { Key = targetKey, Exception = ex }; log.Warn($"Killing peer due to the previous exception"); await input.RetireAsync(); } if (req.Response != null) { try { await req.Response.WriteAsync(res); } catch (Exception ex) { log.Warn("Failed to send response", ex); } } })); }); } } finally { await remotehandler.RetireAsync(); if (connection != null) { try { await connection.ShutdownAsync(); } catch (Exception ex) { log.Warn("Failed to shut down IPC Peer", ex); } } // Write a registration request to the broker await Channels.ConnectionBrokerRegistrations.Get().WriteAsync( new ConnectionRegistrationRequest() { IsTerminate = false, UpdateRouting = false, Peer = remote } ); } }
/// <summary> /// Adds a peer to the routing table /// </summary> /// <returns><c>true</c> if the item was added; <c>false</c> otherwise.</returns> /// <param name="peer">Peer.</param> public bool Add(PeerInfo peer) { return(Add(peer, out var isNew)); }
/// <summary> /// Runs the peer process /// </summary> /// <returns>An awaitable task.</returns> /// <param name="selfinfo">The information describing the peer.</param> /// <param name="k">The redundancy parameter.</param> /// <param name="storesize">The maximum size of the local store</param> /// <param name="maxage">The maximum age of items in the cache</param> /// <param name="initialContactlist">Initial list of peers to contact.</param> /// <param name="requests">The request channel for local management</param> public static async Task RunPeer(PeerInfo selfinfo, int k, int storesize, TimeSpan maxage, EndPoint[] initialContactlist, IReadChannel <PeerRequest> requests) { try { if (selfinfo == null) { throw new ArgumentNullException(nameof(selfinfo)); } if (initialContactlist == null) { throw new ArgumentNullException(nameof(initialContactlist)); } var ip = selfinfo.Address as IPEndPoint; if (ip == null) { throw new ArgumentException($"Unable to convert {nameof(selfinfo.Address)} to a {nameof(IPEndPoint)}", nameof(selfinfo)); } log.Debug($"Starting a peer with key {selfinfo.Key} and address {selfinfo.Address}"); using (var scope = new IsolatedChannelScope()) { var sock = new TcpListener(ip); sock.Start(); // Set up the helper processes var router = RoutingProcess.RunAsync(selfinfo, k); var broker = ConnectionBroker.RunAsync(selfinfo); var values = MRUProcess.RunAsync(selfinfo, storesize, maxage); var remoter = RemoteProcess.RunAsync(selfinfo); log.Debug("Started router, broker, and value store"); // Handle new connections var listener = ListenAsync(selfinfo, sock); log.Debug("Started listener"); // Start discovery of peers var discovery = DiscoveryProcess.RunAsync(selfinfo, initialContactlist); log.Debug("Started discovery"); // The process handling requests to the local node var proc = AutomationExtensions.RunTask( new { Requests = requests, Refresh = Channels.PeerRequests.ForRead, // Add these, so this process terminates the others as well BrokerReg = Channels.ConnectionBrokerRegistrations.ForWrite, BrokerReq = Channels.ConnectionBrokerRequests.ForWrite, BrokerStat = Channels.ConnectionBrokerStats.ForWrite, MRUReq = Channels.MRURequests.ForWrite, MRUStat = Channels.MRUStats.ForWrite, RouteReq = Channels.RoutingTableRequests.ForWrite, RouteStat = Channels.RoutingTableStats.ForWrite, }, async self => { log.Debug("Running peer main loop"); try { while (true) { var req = (await MultiChannelAccess.ReadFromAnyAsync(self.Requests, self.Refresh)).Value; log.Debug($"Peer {selfinfo.Key} got message: {req.Operation}"); switch (req.Operation) { case PeerOperation.Add: await HandleAddOperation(selfinfo, req, k); break; case PeerOperation.Find: await HandleFindOperation(selfinfo, req, k); break; case PeerOperation.Stats: await HandleStatsOperation(req); break; case PeerOperation.Refresh: await HandleRefreshOperation(selfinfo, req, k); break; default: await req.Response.WriteAsync(new PeerResponse() { SuccessCount = -1 }); break; } log.Debug($"Peer {selfinfo.Key} handled message: {req.Operation}"); } } catch (Exception ex) { if (!ex.IsRetiredException()) { log.Warn($"Terminating peer {selfinfo.Key} due to error", ex); } throw; } finally { log.Debug($"Terminating peer {selfinfo.Key}"); } } ); log.Debug("Started main handler"); // Set up a process that periodically emits refresh operations var refresher = AutomationExtensions.RunTask( new { Control = Channels.PeerRequests.ForWrite }, async self => { var respchan = Channel.Create <PeerResponse>(); while (true) { // Sleep, but exit if the parent does if (await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(10)), proc) == proc) { return; } await self.Control.WriteAsync(new PeerRequest() { Operation = PeerOperation.Refresh, Response = respchan }); await respchan.ReadAsync(); } } ); log.Debug("Started refresh process, peer is now live"); await proc; await router; await broker; await values; await remoter; await discovery; await refresher; await Task.WhenAll(router, broker, values, remoter, discovery, refresher); } } catch (Exception ex) { log.Warn("Failed to start peer", ex); try { await requests.RetireAsync(); } catch (Exception ex2) { log.Warn("Failed to stop the input channel", ex2); } log.Debug($"Peer with key {selfinfo.Key} and address {selfinfo.Address} stopped..."); throw; } }
/// <summary> /// Runs the console interface /// </summary> /// <returns>An awaitable task.</returns> public static Task RunAsync() { // Set up a console forwarder process var consoleOut = Skeletons.CollectAsync( Channels.ConsoleOutput.ForRead, x => Console.Out.WriteLineAsync(x ?? string.Empty) ); // Set up a channel for sending control messages var inputChannel = Channel.Create <string>(buffersize: 10); // Set up the console reader process var consoleInput = AutomationExtensions.RunTask( new { Control = inputChannel.AsWrite() }, async self => { string line; // TODO: The blocking read prevents clean shutdown, // but direct access to the input stream has issues with the buffer while ((line = await Task.Run(() => Console.ReadLine())) != null) { await self.Control.WriteAsync(line); } } ); // Set up the control logic handler var proc = AutomationExtensions.RunTask(new { Control = inputChannel.AsRead(), Output = Channels.ConsoleOutput.ForWrite }, async self => { var peers = new List <Tuple <PeerInfo, Task, IWriteChannel <PeerRequest> > >(); var rnd = new Random(); var portnr = 15000; await self.Output.WriteAsync(HELPTEXT); while (true) { try { var commandline = await self.Control.ReadAsync() ?? string.Empty; var command = commandline.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault() ?? string.Empty; if (string.Equals(command, "help", StringComparison.OrdinalIgnoreCase)) { await self.Output.WriteAsync(HELPTEXT); } else if (string.Equals(command, "exit", StringComparison.OrdinalIgnoreCase) || string.Equals(command, "quit", StringComparison.OrdinalIgnoreCase)) { return; } else if (string.Equals(command, "check", StringComparison.OrdinalIgnoreCase)) { for (var i = peers.Count - 1; i >= 0; i--) { if (await peers[i].Item3.IsRetiredAsync) { await self.Output.WriteAsync($"Peer {peers[i].Item1.Key} at {peers[i].Item1.Address} terminated"); peers.RemoveAt(i); } } await self.Output.WriteAsync($"Completed check, found {peers.Count} live peers"); } else if (string.Equals(command, "node", StringComparison.OrdinalIgnoreCase)) { var actions = commandline.Split(new char[] { ' ' }, 4, StringSplitOptions.RemoveEmptyEntries); if (string.Equals(actions[1], "start", StringComparison.OrdinalIgnoreCase)) { var pi = new PeerInfo(Key.CreateRandomKey(), new IPEndPoint(IPAddress.Loopback, portnr)); await self.Output.WriteAsync($"Starting node {pi.Key} on {pi.Address}"); var chan = Channel.Create <PeerRequest>(); var s = Task.Run(() => Peer.RunPeer( pi, 5, 100, TimeSpan.FromDays(1), peers.Count == 0 ? new EndPoint[0] : new[] { peers[rnd.Next(0, peers.Count - 1)].Item1.Address }, chan.AsRead() ) .ContinueWith(_ => inputChannel.WriteAsync("check")) ); peers.Add(new Tuple <PeerInfo, Task, IWriteChannel <PeerRequest> >(pi, s, chan)); portnr++; } else if (string.Equals(actions[1], "list", StringComparison.OrdinalIgnoreCase)) { if (actions.Length != 2) { await self.Output.WriteAsync("The list command takes no arguments"); continue; } for (var i = 0; i < peers.Count; i++) { await self.Output.WriteAsync(string.Format("{0}: {1} - {2}", i, peers[i].Item1.Key, peers[i].Item1.Address)); } await self.Output.WriteAsync(string.Empty); } else if (string.Equals(actions[1], "connect", StringComparison.OrdinalIgnoreCase)) { actions = commandline.Split(new char[] { ' ' }, 5, StringSplitOptions.RemoveEmptyEntries); if (actions.Length != 4) { await self.Output.WriteAsync("The connect command needs exactly two arguments, the ip and the port"); continue; } if (!IPAddress.TryParse(actions[2], out var ip)) { await self.Output.WriteAsync($"Failed to parse ip: {actions[2]}"); continue; } if (!int.TryParse(actions[3], out var port)) { await self.Output.WriteAsync($"Failed to parse {actions[3]} as an integer"); continue; } var pi = new PeerInfo(Key.CreateRandomKey(), new IPEndPoint(IPAddress.Loopback, portnr)); await self.Output.WriteAsync($"Starting node {pi.Key} on {pi.Address}"); var chan = Channel.Create <PeerRequest>(); var s = Task.Run(() => Peer.RunPeer( pi, 5, 100, TimeSpan.FromDays(1), new[] { new IPEndPoint(ip, port) }, chan.AsRead() ) .ContinueWith(_ => inputChannel.WriteAsync("check")) ); peers.Add(new Tuple <PeerInfo, Task, IWriteChannel <PeerRequest> >(pi, s, chan)); portnr++; } else if (string.Equals(actions[1], "stop", StringComparison.OrdinalIgnoreCase) || string.Equals(actions[1], "stat", StringComparison.OrdinalIgnoreCase) || string.Equals(actions[1], "refresh", StringComparison.OrdinalIgnoreCase)) { if (actions.Length != 3) { await self.Output.WriteAsync($"The {actions[1]} command takes exactly one argument, the node number"); continue; } if (!int.TryParse(actions[2], out var ix)) { await self.Output.WriteAsync($"Failed to parse {actions[2]} as an integer"); continue; } if (ix < 0 || ix >= peers.Count) { await self.Output.WriteAsync($"The node number must be positive and less than {peers.Count}"); continue; } if (string.Equals(actions[1], "stop", StringComparison.OrdinalIgnoreCase)) { await self.Output.WriteAsync($"Stopping node {ix} ({peers[ix].Item1.Key} at {peers[ix].Item1.Address}) ..."); await peers[ix].Item3.RetireAsync(); await self.Output.WriteAsync($"Stopped node ({peers[ix].Item1.Key} at {peers[ix].Item1.Address}) ..."); //peers.RemoveAt(ix); } else if (string.Equals(actions[1], "stat", StringComparison.OrdinalIgnoreCase)) { await self.Output.WriteAsync($"Requesting stats from node {ix} ({peers[ix].Item1.Key} at {peers[ix].Item1.Address}) ..."); var channel = Channel.Create <PeerResponse>(); await peers[ix].Item3.WriteAsync(new PeerRequest() { Operation = PeerOperation.Stats, Response = channel }); await self.Output.WriteAsync($"Stats requested, waiting for response..."); await self.Output.WriteAsync(System.Text.Encoding.UTF8.GetString((await channel.ReadAsync()).Data)); } else if (string.Equals(actions[1], "refresh", StringComparison.OrdinalIgnoreCase)) { await self.Output.WriteAsync($"Performing refresh on {ix} ({peers[ix].Item1.Key} at {peers[ix].Item1.Address}) ..."); var channel = Channel.Create <PeerResponse>(); await peers[ix].Item3.WriteAsync(new PeerRequest() { Operation = PeerOperation.Refresh, Response = channel }); var res = await channel.ReadAsync(); await self.Output.WriteAsync($"Refreshed with {res.SuccessCount} node(s)"); } else { await self.Output.WriteAsync($"Node action not recognized: {actions[1]}"); } } else { await self.Output.WriteAsync($"Node command not recognized: {actions[1]}"); } } else if (string.Equals(command, "add", StringComparison.OrdinalIgnoreCase)) { var actions = commandline.Split(new char[] { ' ' }, 2, StringSplitOptions.RemoveEmptyEntries); if (actions.Length == 1) { await self.Output.WriteAsync("The add command needs the value to add"); continue; } if (peers.Count == 0) { await self.Output.WriteAsync("The add command does not work if no nodes are started"); continue; } var channel = Channel.Create <PeerResponse>(); var data = System.Text.Encoding.UTF8.GetBytes(actions[1]); var key = Key.ComputeKey(data); await self.Output.WriteAsync($"Adding {data.Length} byte(s) with key {key}"); await peers[rnd.Next(0, peers.Count)].Item3.WriteAsync(new PeerRequest() { Operation = PeerOperation.Add, Key = key, Data = data, Response = channel, }); await self.Output.WriteAsync("Send add request, waiting for completion"); var res = await channel.ReadAsync(); await self.Output.WriteAsync($"Add inserted into {res.SuccessCount} node(s)"); } else if (string.Equals(command, "get", StringComparison.OrdinalIgnoreCase)) { var actions = commandline.Split(new char[] { ' ' }, 3, StringSplitOptions.RemoveEmptyEntries); if (actions.Length == 1) { await self.Output.WriteAsync("The get command needs the hash to find"); continue; } if (actions.Length == 3) { await self.Output.WriteAsync("The get command needs only one argument"); continue; } if (peers.Count == 0) { await self.Output.WriteAsync("The get command does not work if no nodes are started"); continue; } Key key; try { key = new Key(actions[1]); } catch (Exception ex) { await self.Output.WriteAsync($"Failed to parse key: {ex.Message}"); continue; } var channel = Channel.Create <PeerResponse>(); await self.Output.WriteAsync($"Locating key"); await peers[rnd.Next(0, peers.Count)].Item3.WriteAsync(new PeerRequest() { Operation = PeerOperation.Find, Key = key, Response = channel, }); var res = await channel.ReadAsync(); if (res.Data == null) { await self.Output.WriteAsync($"Did not find the key ..."); } else { await self.Output.WriteAsync($"Found: {System.Text.Encoding.UTF8.GetString(res.Data)}"); } } else if (string.Equals(command, "hash", StringComparison.OrdinalIgnoreCase)) { var actions = commandline.Split(new char[] { ' ' }, 2, StringSplitOptions.RemoveEmptyEntries); if (actions.Length == 1) { await self.Output.WriteAsync("The add command needs the value to add"); continue; } await self.Output.WriteAsync($"Key: {Key.ComputeKey(actions[1])}"); } else { await self.Output.WriteAsync($"Command not recognized: {command}"); } } catch (Exception ex) { await self.Output.WriteAsync($"Command failed: {ex.Message}"); } } } ); return(Task.WhenAll(consoleOut /*, consoleInput*/, proc)); }
/// <summary> /// Runs the MRU cache /// </summary> /// <returns>An awaitable task.</returns> /// <param name="selfinfo">This peer's information</param> /// <param name="storesize">The size of the MRU store</param> /// <param name="maxage">The maximum amount of time items are stored</param> /// <param name="buffersize">The size of the parallel processing buffer</param> private static Task RunMRUAsync(PeerInfo selfinfo, int storesize, TimeSpan maxage, int buffersize) { var storechan = Channel.Create <MRUInternalStore>(); return(AutomationExtensions.RunTask(new { Request = Channels.MRURequests.ForRead, Routing = Channels.RoutingTableRequests.ForWrite, Stats = Channels.MRUStats.ForRead, Store = storechan.AsRead() }, async self => { var cache = new MRUCache <Key, byte[]>(storesize, maxage); var store = new MRUCache <Key, byte[]>(int.MaxValue, maxage); log.Debug($"Store is now running"); // Set up a shared error handler for logging and reporting errors Func <MRURequest, Exception, Task> errorHandler = async(req, ex) => { log.Warn("Failed to process request, sending error", ex); try { await req.SendResponseAsync(ex); } catch (Exception ex2) { log.Warn("Failed to forward error report", ex2); } }; using (var tp = new TaskPool <MRURequest>(buffersize, errorHandler)) while (true) { log.Debug($"Store is waiting for requests ..."); var mreq = await MultiChannelAccess.ReadFromAnyAsync( self.Stats.RequestRead(), self.Store.RequestRead(), self.Request.RequestRead() ); if (mreq.Channel == self.Stats) { log.Debug($"Store got stat request"); var r = (IWriteChannel <MRUStatResponse>)mreq.Value; await r.WriteAsync(new MRUStatResponse() { Items = cache.Count + store.Count, Oldest = new DateTime(Math.Min(cache.OldestItem.Ticks, store.OldestItem.Ticks)), Size = cache.Select(x => x.Value.Length).Sum() + store.Select(x => x.Value.Length).Sum(), Stats = (Channels.MRURequests.Get() as ProfilingChannel <MRURequest>)?.ReportStats() }); continue; } if (mreq.Channel == self.Store) { var sreq = (MRUInternalStore)mreq.Value; log.Debug($"Store got internal store request"); var shouldBroadCast = sreq.Peers != null && !store.TryGetValue(sreq.Key, out _); store.Add(sreq.Key, sreq.Data); // We currently rely on the injector to broadcast, // If we enable this, we need some logic to figure out // the source of the Add, to both allow re-insertion // and avoid repeated broadcasts if two peers determine // they are *the* handling peer //if (shouldBroadCast) //await tp.Run(new MRURequest() { }, () => BroadcastValueAsync(selfinfo, sreq)); continue; } var req = (MRURequest)mreq.Value; log.Debug($"Store got request {req.Operation}"); try { switch (req.Operation) { case MRUOperation.Add: { // Always store it in our cache cache.Add(req.Key, req.Data); // Process long-term if needed await tp.Run(req, () => StoreLongTermAsync(selfinfo, self.Routing, storechan.AsWrite(), req.Key, req.Data)); // Respond that we completed await tp.Run(req, () => req.SendResponseAsync(req.Key, null)); break; } case MRUOperation.Get: { var res = cache.TryGetValue(req.Key, out var data); if (!res) { res = store.TryGetValue(req.Key, out data); } await tp.Run(req, () => req.SendResponseAsync(req.Key, data, res)); break; } case MRUOperation.Expire: cache.ExpireOldItems(); store.ExpireOldItems(); await tp.Run(req, () => req.SendResponseAsync(null, null)); break; default: throw new Exception($"Unable to handle request with type {req.Operation}"); } log.Debug($"Store completed request {req.Operation}"); } catch (Exception ex) { await errorHandler(req, ex); } } })); }