public void TestTimeoutMultiple()
{
var c1 = ChannelManager.CreateChannel <int>();
var c2 = ChannelManager.CreateChannel <int>();
var c3 = ChannelManager.CreateChannel <int>();
Func <Task> p = async() =>
{
try
{
await MultiChannelAccess.ReadFromAnyAsync(TimeSpan.FromSeconds(2), c1, c2, c3);
throw new Exception("Timeout did not happen?");
}
catch (TimeoutException)
{
}
};
var t = p();
if (!t.Wait(TimeSpan.FromSeconds(3)))
{
throw new Exception("Failed to get timeout");
}
}
public void TestMultiTypeReadWrite()
{
var c1 = ChannelManager.CreateChannel <int>();
var c2 = ChannelManager.CreateChannel <string>();
c1.WriteNoWait(1);
c2.WriteNoWait("2");
var r = MultiChannelAccess.ReadFromAnyAsync(c1.AsUntyped(), c2.AsUntyped()).WaitForTask().Result;
if (r == null)
{
throw new Exception("Unexpected null result");
}
if (r.Channel != c1)
{
throw new Exception("Unexpected read channel");
}
if (!(r.Value is int))
{
throw new Exception("Priority changed?");
}
if ((int)r.Value != 1)
{
throw new Exception("Bad value?");
}
r = MultiChannelAccess.ReadFromAnyAsync(c1.RequestRead(), c2.RequestRead()).WaitForTask().Result;
if (r == null)
{
throw new Exception("Unexpected null result");
}
if (r.Channel != c2)
{
throw new Exception("Unexpected read channel");
}
if (!(r.Value is string))
{
throw new Exception("Priority changed?");
}
if ((string)r.Value != "2")
{
throw new Exception("Bad value?");
}
var t = new [] { c1.AsUntyped().RequestWrite(4) }.WriteToAnyAsync();
if (c1.Read() != 4)
{
throw new Exception("Bad value?");
}
t.WaitForTask().Wait();
}
public void TestInvalidMultiAccessOperation()
{
Assert.Throws <InvalidOperationException>(() =>
{
try
{
var c1 = ChannelManager.CreateChannel <int>();
MultiChannelAccess.ReadOrWriteAnyAsync(MultisetRequest.Read(c1), MultisetRequest.Write(1, c1)).WaitForTask().Wait();
}
catch (AggregateException aex)
{
if (aex.InnerExceptions.Count == 1)
{
throw aex.InnerExceptions.First();
}
throw;
}
});
}
public void TestMultiAccessOperation()
{
var c1 = ChannelManager.CreateChannel <int>();
var c2 = ChannelManager.CreateChannel <int>();
// Copy c2 + 1 => c1
Func <Task> p1 = async() => {
var val = await c2.ReadAsync();
while (true)
{
var res = await MultiChannelAccess.ReadOrWriteAnyAsync(MultisetRequest.Read(c2), MultisetRequest.Write(val, c1));
if (res.IsRead)
{
val = res.Value + 1;
}
}
};
// Copy c1 => c2
Func <Task> p2 = async() => {
var val = 1;
for (var i = 0; i < 10; i++)
{
await c2.WriteAsync(val);
val = await c1.ReadAsync();
}
c1.Retire();
c2.Retire();
if (val != 10)
{
throw new InvalidProgramException("Bad counter!");
}
};
// Wait for shutdown
try
{
Task.WhenAll(p1(), p2()).WaitForTask().Wait();
}
catch (Exception ex)
{
// Filter out all ChannelRetired exceptions
if (ex is AggregateException)
{
var rex = (from n in (ex as AggregateException).InnerExceptions
where !(n is RetiredException)
select n);
if (rex.Count() == 1)
{
throw rex.First();
}
else if (rex.Count() != 0)
{
throw new AggregateException(rex);
}
}
else
{
throw;
}
}
}
/// <summary>
/// Runs the broker process.
/// </summary>
/// <param name="node">This nodes information</param>
/// <param name="maxconnections">The maximum number of connections to allow</param>
/// <returns>An awaitable task.</returns>
public static Task RunAsync(PeerInfo node, int maxconnections = 50)
{
// The primary table for finding peers
var peers = new Dictionary <EndPoint, Tuple <Task, IWriteChannel <ConnectionRequest> > >();
// The peers listed by key
var peersbykey = new Dictionary <Key, EndPoint>();
// The MRU cache of peers
var mrucache = new MRUCache <EndPoint, Key>(maxconnections, TimeSpan.FromDays(10));
return(AutomationExtensions.RunTask(
new
{
Request = Channels.ConnectionBrokerRequests.ForRead,
Registrations = Channels.ConnectionBrokerRegistrations.ForRead,
Stats = Channels.ConnectionBrokerStats.ForRead,
SelfHandler = Channels.RemoteRequests.ForWrite,
Routing = Channels.RoutingTableRequests.ForWrite
},
async self =>
{
log.Debug($"Broker is now running");
while (true)
{
log.Debug($"Broker is waiting for requests ...");
var mreq = await MultiChannelAccess.ReadFromAnyAsync(
self.Stats.RequestRead(),
self.Registrations.RequestRead(),
self.Request.RequestRead()
);
if (mreq.Channel == self.Stats)
{
log.Debug($"Broker got stat request");
var req = (IWriteChannel <ConnectionStatsResponse>)mreq.Value;
await req.WriteAsync(new ConnectionStatsResponse()
{
EndPoints = peers.Count,
Keys = peersbykey.Count,
Stats = (Channels.ConnectionBrokerRequests.Get() as ProfilingChannel <ConnectionRequest>)?.ReportStats()
});
}
else if (mreq.Channel == self.Registrations)
{
var req = (ConnectionRegistrationRequest)mreq.Value;
log.Debug($"Broker got {(req.IsTerminate ? "termination" : "registration")} request");
if (req.IsTerminate)
{
// Make sure we do not have stale stuff in the MRU cache
if (req.Peer != null && req.Peer.Address != null)
{
mrucache.Remove(req.Peer.Address);
}
if (req.Peer.Address != null && peers.TryGetValue(req.Peer.Address, out var c) && c.Item2 == req.Channel)
{
peers.Remove(req.Peer.Address);
if (req.Peer.Key != null)
{
peersbykey.Remove(req.Peer.Key);
}
}
if (req.UpdateRouting)
{
log.Debug($"Removing peer in routing table due to termination of connection {req.Peer.Key} - {req.Peer.Address}");
await self.Routing.RemovePeerAsync(req.Peer.Key);
}
}
else
{
if (req.Peer.Address != null && peers.TryGetValue(req.Peer.Address, out var c) && (c.Item2 == req.Channel || c == null))
{
if (c == null)
{
peers[req.Peer.Address] = new Tuple <Task, IWriteChannel <ConnectionRequest> >(null, req.Channel);
}
if (!peersbykey.ContainsKey(req.Peer.Key))
{
peersbykey[req.Peer.Key] = req.Peer.Address;
}
}
if (req.UpdateRouting)
{
log.Debug($"Adding new peer to routing table {req.Peer.Key} - {req.Peer.Address}");
await self.Routing.AddPeerAsync(req.Peer.Key, req.Peer);
}
}
}
else
{
var req = (ConnectionRequest)mreq.Value;
log.Debug($"Broker got connection request for {req.EndPoint}");
// Check if we request ourselves
if (node.Key.Equals(req.Key) || node.Address.Equals(req.EndPoint))
{
log.Debug($"Broker got self-request, forwarding to owner");
await self.SelfHandler.WriteAsync(req);
continue;
}
Tuple <Task, IWriteChannel <ConnectionRequest> > peer = null;
try
{
// Existing connection, update MRU
var overflow = mrucache.Add(req.EndPoint, req.Key);
// If we have too many connections, kill one now
if (overflow != null)
{
// We could make this also take the closest k peers into account
log.Debug($"Broker has too many connections, closing {req.EndPoint}");
await peers[overflow].Item2.RetireAsync();
}
if (!peers.TryGetValue(req.EndPoint, out peer))
{
log.Debug($"Broker is starting a connection to {req.EndPoint}");
mrucache.Add(req.EndPoint, req.Key);
peer = peers[req.EndPoint] =
PeerConnection.CreatePeer(
node,
new PeerInfo(req.Key, req.EndPoint),
() => ConnectToPeerAsync(req.EndPoint),
REQ_BUFFER_SIZE
);
if (req.Key != null)
{
peersbykey[req.Key] = req.EndPoint;
}
}
await peer.Item2.WriteAsync(req);
}
catch (Exception ex)
{
log.Warn("Failed to send request to peer", ex);
try { await req.Response.WriteAsync(new ConnectionResponse()
{
Exception = ex
}); }
catch (Exception ex2) { log.Warn("Failed to write failure response", ex2); }
if (peer != null)
{
try { peer.Item2.AsWriteOnly().Dispose(); }
catch (Exception ex2) { log.Warn("Failed to terminate write channel", ex2); }
try { await peer.Item1; }
catch (Exception ex2) { log.Warn("Peer connection stopped with error", ex2); }
}
peers.Remove(req.EndPoint);
}
}
}
}
));
}
/// <summary>
/// Runs the router and forwarder
/// </summary>
/// <returns>An awaitable result.</returns>
/// <param name="owner">The owner of the routing table.</param>
/// <param name="k">The redundancy parameter.</param>
/// <param name="buffersize">The size of the forwarding buffer.</param>
public static Task RunAsync(PeerInfo owner, int k, int buffersize = 10)
{
return(AutomationExtensions.RunTask(
new
{
Requests = Channels.RoutingTableRequests.ForRead,
Stats = Channels.RoutingTableStats.ForRead,
PeerReq = Channels.PeerRequests.ForWrite
},
async self =>
{
// Setup the routing table, and add us to it
var table = new RoutingTable(owner.Key, k);
table.Add(owner);
log.Debug($"Router is now running");
// Set up the error handler
Func <RoutingRequest, Exception, Task> errorHandler = async(t, ex) =>
{
log.Warn("Routing operation failed, sending failure response to requester", ex);
try
{
await t.Response.WriteAsync(new RoutingResponse()
{
Exception = ex, Succes = false
});
}
catch (Exception ex2)
{
log.Warn("Failed to forward error message", ex2);
}
};
using (var tp = new TaskPool <RoutingRequest>(buffersize, errorHandler))
while (true)
{
// Wait for requests
log.Debug($"Router is waiting for requests ...");
var r = await MultiChannelAccess.ReadFromAnyAsync(self.Stats.RequestRead(), self.Requests.RequestRead());
if (r.Channel == self.Stats)
{
log.Debug($"Router got stat request");
var m = (IWriteChannel <RoutingStatsResponse>)r.Value;
await m.WriteAsync(new RoutingStatsResponse()
{
Count = table.Count,
Stats = (Channels.RoutingTableRequests.Get() as ProfilingChannel <RoutingRequest>)?.ReportStats()
});
continue;
}
var data = (RoutingRequest)r.Value;
try
{
log.Debug($"Router got request {data.Operation}");
// Multiplex the operation
switch (data.Operation)
{
case RoutingOperation.Add:
{
var success = table.Add(data.Data, out var isNew);
if (data.Response != null)
{
await tp.Run(data, () => data.Response.WriteAsync(new RoutingResponse()
{
Succes = true, IsNew = isNew
}));
}
// If the peer is new, discover what peers it knows
if (isNew)
{
log.Debug($"New peer, requesting refresh");
await tp.Run(data, () => self.PeerReq.WriteAsync(new PeerRequest()
{
Operation = PeerOperation.Refresh,
Key = data.Data.Key
}));
log.Debug($"Peer refresh requested");
}
break;
}
case RoutingOperation.Remove:
{
var sucess = table.RemoveKey(data.Key);
if (data.Response != null)
{
await tp.Run(data, () => data.Response.WriteAsync(new RoutingResponse()
{
Succes = sucess
}));
}
break;
}
case RoutingOperation.Lookup:
{
var peers = table.Nearest(data.Key, k, data.OnlyKBucket);
await tp.Run(data, () => data.Response.WriteAsync(new RoutingResponse()
{
Succes = true, Peers = peers
}));
break;
}
default:
throw new Exception($"Operation not supported: {data.Operation}");
}
}
catch (Exception ex)
{
await errorHandler(data, ex);
}
}
}));
}
/// <summary>
/// Runs the MRU cache
/// </summary>
/// <returns>An awaitable task.</returns>
/// <param name="selfinfo">This peer's information</param>
/// <param name="storesize">The size of the MRU store</param>
/// <param name="maxage">The maximum amount of time items are stored</param>
/// <param name="buffersize">The size of the parallel processing buffer</param>
private static Task RunMRUAsync(PeerInfo selfinfo, int storesize, TimeSpan maxage, int buffersize)
{
var storechan = Channel.Create <MRUInternalStore>();
return(AutomationExtensions.RunTask(new
{
Request = Channels.MRURequests.ForRead,
Routing = Channels.RoutingTableRequests.ForWrite,
Stats = Channels.MRUStats.ForRead,
Store = storechan.AsRead()
},
async self =>
{
var cache = new MRUCache <Key, byte[]>(storesize, maxage);
var store = new MRUCache <Key, byte[]>(int.MaxValue, maxage);
log.Debug($"Store is now running");
// Set up a shared error handler for logging and reporting errors
Func <MRURequest, Exception, Task> errorHandler = async(req, ex) =>
{
log.Warn("Failed to process request, sending error", ex);
try { await req.SendResponseAsync(ex); }
catch (Exception ex2) { log.Warn("Failed to forward error report", ex2); }
};
using (var tp = new TaskPool <MRURequest>(buffersize, errorHandler))
while (true)
{
log.Debug($"Store is waiting for requests ...");
var mreq = await MultiChannelAccess.ReadFromAnyAsync(
self.Stats.RequestRead(),
self.Store.RequestRead(),
self.Request.RequestRead()
);
if (mreq.Channel == self.Stats)
{
log.Debug($"Store got stat request");
var r = (IWriteChannel <MRUStatResponse>)mreq.Value;
await r.WriteAsync(new MRUStatResponse()
{
Items = cache.Count + store.Count,
Oldest = new DateTime(Math.Min(cache.OldestItem.Ticks, store.OldestItem.Ticks)),
Size = cache.Select(x => x.Value.Length).Sum() + store.Select(x => x.Value.Length).Sum(),
Stats = (Channels.MRURequests.Get() as ProfilingChannel <MRURequest>)?.ReportStats()
});
continue;
}
if (mreq.Channel == self.Store)
{
var sreq = (MRUInternalStore)mreq.Value;
log.Debug($"Store got internal store request");
var shouldBroadCast = sreq.Peers != null && !store.TryGetValue(sreq.Key, out _);
store.Add(sreq.Key, sreq.Data);
// We currently rely on the injector to broadcast,
// If we enable this, we need some logic to figure out
// the source of the Add, to both allow re-insertion
// and avoid repeated broadcasts if two peers determine
// they are *the* handling peer
//if (shouldBroadCast)
//await tp.Run(new MRURequest() { }, () => BroadcastValueAsync(selfinfo, sreq));
continue;
}
var req = (MRURequest)mreq.Value;
log.Debug($"Store got request {req.Operation}");
try
{
switch (req.Operation)
{
case MRUOperation.Add:
{
// Always store it in our cache
cache.Add(req.Key, req.Data);
// Process long-term if needed
await tp.Run(req, () => StoreLongTermAsync(selfinfo, self.Routing, storechan.AsWrite(), req.Key, req.Data));
// Respond that we completed
await tp.Run(req, () => req.SendResponseAsync(req.Key, null));
break;
}
case MRUOperation.Get:
{
var res = cache.TryGetValue(req.Key, out var data);
if (!res)
{
res = store.TryGetValue(req.Key, out data);
}
await tp.Run(req, () => req.SendResponseAsync(req.Key, data, res));
break;
}
case MRUOperation.Expire:
cache.ExpireOldItems();
store.ExpireOldItems();
await tp.Run(req, () => req.SendResponseAsync(null, null));
break;
default:
throw new Exception($"Unable to handle request with type {req.Operation}");
}
log.Debug($"Store completed request {req.Operation}");
}
catch (Exception ex)
{
await errorHandler(req, ex);
}
}
}));
}
public void TestMultiTypeReadWrite()
{
var c1 = ChannelManager.CreateChannel <int>();
var c2 = ChannelManager.CreateChannel <string>();
var c3 = ChannelManager.CreateChannel <long>();
c1.WriteNoWait(1);
c2.WriteNoWait("2");
c3.WriteNoWait(3);
// Using explicit .AsUntyped() calls
var r = MultiChannelAccess.ReadFromAnyAsync(c1.AsUntyped(), c2.AsUntyped(), c3.AsUntyped()).WaitForTask().Result;
if (r == null)
{
throw new UnittestException("Unexpected null result");
}
if (r.Channel != c1)
{
throw new UnittestException("Unexpected read channel");
}
if (!(r.Value is int))
{
throw new UnittestException("Priority changed?");
}
if ((int)r.Value != 1)
{
throw new UnittestException("Bad value?");
}
// Using explicit .RequestRead() calls
r = MultiChannelAccess.ReadFromAnyAsync(c1.RequestRead(), c2.RequestRead(), c3.RequestRead()).WaitForTask().Result;
if (r == null)
{
throw new UnittestException("Unexpected null result");
}
if (r.Channel != c2)
{
throw new UnittestException("Unexpected read channel");
}
if (!(r.Value is string))
{
throw new UnittestException("Priority changed?");
}
if ((string)r.Value != "2")
{
throw new UnittestException("Bad value?");
}
// Using channels directly
r = MultiChannelAccess.ReadFromAnyAsync(c1, c2, c3).WaitForTask().Result;
if (r == null)
{
throw new UnittestException("Unexpected null result");
}
if (r.Channel != c3)
{
throw new UnittestException("Unexpected read channel");
}
if (!(r.Value is long))
{
throw new UnittestException("Priority changed?");
}
if ((long)r.Value != 3)
{
throw new UnittestException("Bad value?");
}
// Writing with untyped channel request
var t = new [] { c1.AsUntyped().RequestWrite(4) }.WriteToAnyAsync();
if (c1.Read() != 4)
{
throw new UnittestException("Bad value?");
}
t.WaitForTask().Wait();
// Writing with a typed write request, using mixed types
var t2 = MultiChannelAccess.WriteToAnyAsync(c1.RequestWrite(5), c2.RequestWrite("6"));
if (c1.Read() != 5)
{
throw new UnittestException("Bad value?");
}
t2.WaitForTask().Wait();
}
/// <summary>
/// Runs the peer process
/// </summary>
/// <returns>An awaitable task.</returns>
/// <param name="selfinfo">The information describing the peer.</param>
/// <param name="k">The redundancy parameter.</param>
/// <param name="storesize">The maximum size of the local store</param>
/// <param name="maxage">The maximum age of items in the cache</param>
/// <param name="initialContactlist">Initial list of peers to contact.</param>
/// <param name="requests">The request channel for local management</param>
public static async Task RunPeer(PeerInfo selfinfo, int k, int storesize, TimeSpan maxage, EndPoint[] initialContactlist, IReadChannel <PeerRequest> requests)
{
try
{
if (selfinfo == null)
{
throw new ArgumentNullException(nameof(selfinfo));
}
if (initialContactlist == null)
{
throw new ArgumentNullException(nameof(initialContactlist));
}
var ip = selfinfo.Address as IPEndPoint;
if (ip == null)
{
throw new ArgumentException($"Unable to convert {nameof(selfinfo.Address)} to a {nameof(IPEndPoint)}", nameof(selfinfo));
}
log.Debug($"Starting a peer with key {selfinfo.Key} and address {selfinfo.Address}");
using (var scope = new IsolatedChannelScope())
{
var sock = new TcpListener(ip);
sock.Start();
// Set up the helper processes
var router = RoutingProcess.RunAsync(selfinfo, k);
var broker = ConnectionBroker.RunAsync(selfinfo);
var values = MRUProcess.RunAsync(selfinfo, storesize, maxage);
var remoter = RemoteProcess.RunAsync(selfinfo);
log.Debug("Started router, broker, and value store");
// Handle new connections
var listener = ListenAsync(selfinfo, sock);
log.Debug("Started listener");
// Start discovery of peers
var discovery = DiscoveryProcess.RunAsync(selfinfo, initialContactlist);
log.Debug("Started discovery");
// The process handling requests to the local node
var proc = AutomationExtensions.RunTask(
new
{
Requests = requests,
Refresh = Channels.PeerRequests.ForRead,
// Add these, so this process terminates the others as well
BrokerReg = Channels.ConnectionBrokerRegistrations.ForWrite,
BrokerReq = Channels.ConnectionBrokerRequests.ForWrite,
BrokerStat = Channels.ConnectionBrokerStats.ForWrite,
MRUReq = Channels.MRURequests.ForWrite,
MRUStat = Channels.MRUStats.ForWrite,
RouteReq = Channels.RoutingTableRequests.ForWrite,
RouteStat = Channels.RoutingTableStats.ForWrite,
},
async self =>
{
log.Debug("Running peer main loop");
try
{
while (true)
{
var req = (await MultiChannelAccess.ReadFromAnyAsync(self.Requests, self.Refresh)).Value;
log.Debug($"Peer {selfinfo.Key} got message: {req.Operation}");
switch (req.Operation)
{
case PeerOperation.Add:
await HandleAddOperation(selfinfo, req, k);
break;
case PeerOperation.Find:
await HandleFindOperation(selfinfo, req, k);
break;
case PeerOperation.Stats:
await HandleStatsOperation(req);
break;
case PeerOperation.Refresh:
await HandleRefreshOperation(selfinfo, req, k);
break;
default:
await req.Response.WriteAsync(new PeerResponse()
{
SuccessCount = -1
});
break;
}
log.Debug($"Peer {selfinfo.Key} handled message: {req.Operation}");
}
}
catch (Exception ex)
{
if (!ex.IsRetiredException())
{
log.Warn($"Terminating peer {selfinfo.Key} due to error", ex);
}
throw;
}
finally
{
log.Debug($"Terminating peer {selfinfo.Key}");
}
}
);
log.Debug("Started main handler");
// Set up a process that periodically emits refresh operations
var refresher = AutomationExtensions.RunTask(
new { Control = Channels.PeerRequests.ForWrite },
async self =>
{
var respchan = Channel.Create <PeerResponse>();
while (true)
{
// Sleep, but exit if the parent does
if (await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(10)), proc) == proc)
{
return;
}
await self.Control.WriteAsync(new PeerRequest()
{
Operation = PeerOperation.Refresh,
Response = respchan
});
await respchan.ReadAsync();
}
}
);
log.Debug("Started refresh process, peer is now live");
await proc;
await router;
await broker;
await values;
await remoter;
await discovery;
await refresher;
await Task.WhenAll(router, broker, values, remoter, discovery, refresher);
}
}
catch (Exception ex)
{
log.Warn("Failed to start peer", ex);
try { await requests.RetireAsync(); }
catch (Exception ex2) { log.Warn("Failed to stop the input channel", ex2); }
log.Debug($"Peer with key {selfinfo.Key} and address {selfinfo.Address} stopped...");
throw;
}
}
