/// <inheritdoc/>
public async Task RegisterAsync(IConsulRegistration registration)
{
if (registration == null)
{
throw new ArgumentNullException(nameof(registration));
}
_logger?.LogInformation("Registering service with consul {serviceId} ", registration.ServiceId);
try
{
await _client.Agent.ServiceRegister(registration.Service);
if (Options.IsHeartBeatEnabled && _scheduler != null)
{
_scheduler.Add(registration.InstanceId);
}
}
catch (Exception e)
{
if (Options.FailFast)
{
_logger?.LogError(e, "Error registering service with consul {serviceId} ", registration.ServiceId);
throw;
}
_logger?.LogWarning(e, "Failfast is false. Error registering service with consul {serviceId} ", registration.ServiceId);
}
}
protected ScheduledService(ISchedule schedule, IScheduler scheduler)
{
Scheduler = scheduler;
Schedule = schedule;
scheduler.Add(schedule, Run);
}
public IEnumerable <IVertex> Crawl(Uri link)
{
//Parse first page
var exploredVertex = _parser.Parse(link);
_scheduler.Add(exploredVertex);
////Next scheduled pages
while (_scheduler.Any())
{
foreach (var unexploredVertex in _scheduler)
{
var friends = _parser.ParseFriends(unexploredVertex);
_scheduler.Add(friends);
_scheduler.Remove(unexploredVertex);
yield return(unexploredVertex);
}
}
}
public async Task <CrawlResult> Crawl(Uri uri, CancellationTokenSource cancellationTokenSource)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
_crawlContext.OriginalRootUri = uri;
if (cancellationTokenSource != null)
{
_crawlContext.CancellationTokenSource = cancellationTokenSource;
}
_crawlResult = new CrawlResult
{
RootUri = _crawlContext.OriginalRootUri,
CrawlContext = _crawlContext
};
var timer = Stopwatch.StartNew();
try
{
var rootPage = new PageToCrawl(uri)
{
ParentUri = uri, IsInternal = true, IsRoot = true
};
_scheduler.Add(rootPage);
await CrawlSite().ConfigureAwait(false);
}
catch (Exception e)
{
_crawlResult.ErrorException = e;
}
finally
{
_threadManager?.Dispose();
}
timer.Stop();
_crawlResult.Elapsed = timer.Elapsed;
return(_crawlResult);
}
private void _trayIcon_MoreOptionsOnClick(object sender, MoreOptionsOnClickEventArgs e)
{
var dialog = new MoreOptionsDialog(_translation);
var result = dialog.ShowDialog(e.Socket);
if (!result.Canceled)
{
_scheduler.Add(new ScheduledTask(
DateTime.Now + result.TimeSpan,
result.Status,
result.Socket));
}
}
static void Main(string[] args)
{
IScheduler scheduler = SchedulerFactory.Create(SchedulerType.RoundRobin);
foreach (string s in Directory.GetFiles(@"C:\Downloads"))
{
scheduler.Add(new CopyFileAsync(s, @"C:\Downloads\COPYTEST\COPYDEST"));
}
scheduler.CoroutineComplete += new CoroutineCompleteHandler(scheduler_CoroutineComplete);
scheduler.Run();
Console.ReadKey();
}
public async Task Crawl(Uri startingUrl, int maxUrlsToCrawl)
{
var maxCrawls = maxUrlsToCrawl;
var currentCrawls = 0;
var crawlComplete = false;
_scheduler.Add(startingUrl);
while (!crawlComplete && currentCrawls < maxCrawls)
{
var linksLeftToSchedule = _scheduler.Count;
if (linksLeftToSchedule > 0)
{
var linkToCrawl = _scheduler.GetNext();
ProcessPage(await _pageRequester.MakeRequest(linkToCrawl));
currentCrawls++;
}
else
{
//Finished the crawl
crawlComplete = true;
}
}
}
internal KRPCServer(IPAddress address, ushort rpcPort, ushort streamPort)
{
rpcTcpServer = new TCPServer ("RPCServer", address, rpcPort);
streamTcpServer = new TCPServer ("StreamServer", address, streamPort);
rpcServer = new RPCServer (rpcTcpServer);
streamServer = new StreamServer (streamTcpServer);
clientScheduler = new RoundRobinScheduler<IClient<Request,Response>> ();
continuations = new List<RequestContinuation> ();
streamRequests = new Dictionary<IClient<byte,StreamMessage>,IList<StreamRequest>> ();
// Tie events to underlying server
rpcServer.OnStarted += (s, e) => {
if (OnStarted != null)
OnStarted (this, EventArgs.Empty);
};
rpcServer.OnStopped += (s, e) => {
if (OnStopped != null)
OnStopped (this, EventArgs.Empty);
};
rpcServer.OnClientRequestingConnection += (s, e) => {
if (OnClientRequestingConnection != null)
OnClientRequestingConnection (s, e);
};
rpcServer.OnClientConnected += (s, e) => {
if (OnClientConnected != null)
OnClientConnected (s, e);
};
rpcServer.OnClientDisconnected += (s, e) => {
if (OnClientDisconnected != null)
OnClientDisconnected (s, e);
};
// Add/remove clients from the scheduler
rpcServer.OnClientConnected += (s, e) => clientScheduler.Add (e.Client);
rpcServer.OnClientDisconnected += (s, e) => clientScheduler.Remove (e.Client);
// Add/remove clients from the list of stream requests
streamServer.OnClientConnected += (s, e) => streamRequests [e.Client] = new List<StreamRequest> ();
streamServer.OnClientDisconnected += (s, e) => streamRequests.Remove (e.Client);
// Validate stream client identifiers
streamServer.OnClientRequestingConnection += (s, e) => {
if (rpcServer.Clients.Where (c => c.Guid == e.Client.Guid).Any ())
e.Request.Allow ();
else
e.Request.Deny ();
};
}
public void EnabledTest()
{
TimeSpan triggerPeriod = new TimeSpan(5, 0, 0);
ISchedule target = theScheduler.Add(triggerPeriod, false);
Assert.IsFalse(target.Enabled);
target.Enabled = true;
DateTime nextDueTime = DateTime.Now + target.Period;
Assert.AreNotEqual(target.NextDueTime, DateTime.MaxValue);
Assert.AreEqual(target.NextDueTime, nextDueTime);
target.Enabled = false;
Assert.AreEqual(target.NextDueTime, DateTime.MaxValue);
}
public void AddTest()
{
ISchedule createdSchedule = theScheduler.Add(new TimeSpan(0, 0, 5), true);
Assert.IsTrue(theScheduler.Contains(createdSchedule), "Created schedule couldn't be found");
Console.WriteLine("Next time schedule is signaled: " + createdSchedule.NextDueTime);
int waitTimeInMilliseconds = Convert.ToInt32(createdSchedule.Period.TotalMilliseconds);
Console.WriteLine("Waiting for: " + waitTimeInMilliseconds + " ms to be signaled.");
if (createdSchedule.ScheduleSignal.WaitOne(waitTimeInMilliseconds * 2, false))
{
Console.WriteLine("Signal was set in expected wait interval.");
}
else
{
Assert.Fail("Signal was not set in expected wait interval.");
}
//We add a new Schedule and look if the timing mechanism is recalculated
ISchedule fastSchedule = theScheduler.Add(new TimeSpan(0, 0, 1), true);
Assert.IsTrue(theScheduler.Contains(createdSchedule), "Created schedule couldn't be found");
if (fastSchedule.ScheduleSignal.WaitOne(1000 * 2, false))
{
Console.WriteLine("Fast signal was set in expected wait interval.");
}
else
{
Assert.Fail("Fast signal was not set in expected wait interval.");
}
theScheduler.Remove(createdSchedule);
theScheduler.Remove(fastSchedule);
}
private async Task RegisterAsyncInternal(IConsulRegistration registration)
{
_logger?.LogInformation("Registering service with consul {serviceId} ", registration.ServiceId);
try
{
await _client.Agent.ServiceRegister(registration.Service).ConfigureAwait(false);
if (Options.IsHeartBeatEnabled && _scheduler != null)
{
_scheduler.Add(registration.InstanceId);
}
}
catch (Exception e)
{
if (Options.FailFast)
{
_logger?.LogError(e, "Error registering service with consul {serviceId} ", registration.ServiceId);
throw;
}
_logger?.LogWarning(e, "Failfast is false. Error registering service with consul {serviceId} ", registration.ServiceId);
}
}
/// <summary>
/// Begins a synchronous crawl using the uri param, subscribe to events to process data as it becomes available
/// </summary>
public virtual async Task <CrawlResult> CrawlAsync(Uri uri, CancellationTokenSource cancellationTokenSource)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
_crawlContext.RootUri = _crawlContext.OriginalRootUri = uri;
if (cancellationTokenSource != null)
{
_crawlContext.CancellationTokenSource = cancellationTokenSource;
}
_crawlResult = new CrawlResult();
_crawlResult.RootUri = _crawlContext.RootUri;
_crawlResult.CrawlContext = _crawlContext;
_crawlComplete = false;
_logger.LogInformation($"About to crawl site [{uri.AbsoluteUri}]");
PrintConfigValues(_crawlContext.CrawlConfiguration);
if (_memoryManager != null)
{
_crawlContext.MemoryUsageBeforeCrawlInMb = _memoryManager.GetCurrentUsageInMb();
_logger.LogInformation($"Starting memory usage for site [{uri.AbsoluteUri}] is [{_crawlContext.MemoryUsageBeforeCrawlInMb}mb]");
}
_crawlContext.CrawlStartDate = DateTime.Now;
var timer = Stopwatch.StartNew();
if (_crawlContext.CrawlConfiguration.CrawlTimeoutSeconds > 0)
{
_timeoutTimer = new Timer(HandleCrawlTimeout, null, 0, _crawlContext.CrawlConfiguration.CrawlTimeoutSeconds * 1000);
}
try
{
var rootPage = new PageToCrawl(uri)
{
ParentUri = uri, IsInternal = true, IsRoot = true
};
if (ShouldSchedulePageLink(rootPage))
{
_scheduler.Add(rootPage);
}
VerifyRequiredAvailableMemory();
await CrawlSite();
}
catch (Exception e)
{
_crawlResult.ErrorException = e;
_logger.LogCritical($"An error occurred while crawling site [{uri}]", e);
}
finally
{
_threadManager?.Dispose();
}
_timeoutTimer?.Dispose();
timer.Stop();
if (_memoryManager != null)
{
_crawlContext.MemoryUsageAfterCrawlInMb = _memoryManager.GetCurrentUsageInMb();
_logger.LogInformation($"Ending memory usage for site [{uri.AbsoluteUri}] is [{_crawlContext.MemoryUsageAfterCrawlInMb}mb]");
}
_crawlResult.Elapsed = timer.Elapsed;
_logger.LogInformation($"Crawl complete for site [{_crawlResult.RootUri.AbsoluteUri}]: Crawled [{_crawlResult.CrawlContext.CrawledCount}] pages in [{_crawlResult.Elapsed}]");
return(_crawlResult);
}
/// <summary>
/// Begins a synchronous crawl using the uri param, subscribe to events to process data as it becomes available
/// </summary>
public virtual CrawlResult Crawl(Uri uri, CancellationTokenSource cancellationTokenSource)
{
if (uri == null)
{
throw new ArgumentNullException("uri");
}
_crawlContext.RootUri = uri;
if (cancellationTokenSource != null)
{
_crawlContext.CancellationTokenSource = cancellationTokenSource;
}
_crawlResult = new CrawlResult();
_crawlResult.RootUri = _crawlContext.RootUri;
_crawlResult.CrawlContext = _crawlContext;
_crawlComplete = false;
_logger.InfoFormat("About to crawl site [{0}]", uri.AbsoluteUri);
PrintConfigValues(_crawlContext.CrawlConfiguration);
if (_memoryManager != null)
{
_crawlContext.MemoryUsageBeforeCrawlInMb = _memoryManager.GetCurrentUsageInMb();
_logger.InfoFormat("Starting memory usage for site [{0}] is [{1}mb]", uri.AbsoluteUri, _crawlContext.MemoryUsageBeforeCrawlInMb);
}
_crawlContext.CrawlStartDate = DateTime.Now;
Stopwatch timer = Stopwatch.StartNew();
if (_crawlContext.CrawlConfiguration.CrawlTimeoutSeconds > 0)
{
_timeoutTimer = new Timer(_crawlContext.CrawlConfiguration.CrawlTimeoutSeconds * 1000);
_timeoutTimer.Elapsed += HandleCrawlTimeout;
_timeoutTimer.Start();
}
try
{
PageToCrawl rootPage = new PageToCrawl(uri)
{
ParentUri = uri, IsInternal = true, IsRoot = true
};
if (ShouldSchedulePageLink(rootPage))
{
_scheduler.Add(rootPage);
}
VerifyRequiredAvailableMemory();
CrawlSite();
}
catch (Exception e)
{
_crawlResult.ErrorException = e;
_logger.FatalFormat("An error occurred while crawling site [{0}]", uri);
_logger.Fatal(e);
}
finally
{
if (_threadManager != null)
{
_threadManager.Dispose();
}
}
if (_timeoutTimer != null)
{
_timeoutTimer.Stop();
}
timer.Stop();
if (_memoryManager != null)
{
_crawlContext.MemoryUsageAfterCrawlInMb = _memoryManager.GetCurrentUsageInMb();
_logger.InfoFormat("Ending memory usage for site [{0}] is [{1}mb]", uri.AbsoluteUri, _crawlContext.MemoryUsageAfterCrawlInMb);
}
_crawlResult.Elapsed = timer.Elapsed;
_logger.InfoFormat("Crawl complete for site [{0}]: Crawled [{1}] pages in [{2}]", _crawlResult.RootUri.AbsoluteUri, _crawlResult.CrawlContext.CrawledCount, _crawlResult.Elapsed);
return(_crawlResult);
}
private void HandleRequest(string request)
{
JToken payload;
try
{
payload = JToken.Parse(request);
}
catch
{
_outputHandler.Send(new ParseError());
return;
}
if (!_receiver.IsValid(payload))
{
_outputHandler.Send(new InvalidRequest());
return;
}
var(requests, hasResponse) = _receiver.GetRequests(payload);
if (hasResponse)
{
foreach (var response in requests.Where(x => x.IsResponse).Select(x => x.Response))
{
var id = response.Id is string s?long.Parse(s) : response.Id is long l ? l : -1;
if (id < 0)
{
continue;
}
var tcs = _responseRouter.GetRequest(id);
if (tcs is null)
{
continue;
}
if (response is ServerResponse serverResponse)
{
tcs.SetResult(serverResponse.Result);
}
else if (response is ServerError serverError)
{
tcs.SetException(new JsonRpcException(serverError));
}
}
return;
}
foreach (var item in requests)
{
if (item.IsRequest)
{
var descriptor = _requestRouter.GetDescriptor(item.Request);
if (descriptor is null)
{
continue;
}
var type = _requestProcessIdentifier.Identify(descriptor);
_requestRouter.StartRequest(item.Request.Id);
_scheduler.Add(
type,
item.Request.Method,
async() => {
try
{
var result = await _requestRouter.RouteRequest(descriptor, item.Request, CancellationToken.None);
if (result.IsError && result.Error is RequestCancelled)
{
return;
}
_outputHandler.Send(result.Value);
}
catch (Exception e)
{
_logger.LogCritical(Events.UnhandledRequest, e, "Unhandled exception executing request {Method}@{Id}", item.Request.Method, item.Request.Id);
// TODO: Should we rethrow or swallow?
// If an exception happens... the whole system could be in a bad state, hence this throwing currently.
throw;
}
}
);
}
if (item.IsNotification)
{
var descriptor = _requestRouter.GetDescriptor(item.Notification);
if (descriptor is null)
{
continue;
}
// We need to special case cancellation so that we can cancel any request that is currently in flight.
if (descriptor.Method == JsonRpcNames.CancelRequest)
{
var cancelParams = item.Notification.Params?.ToObject <CancelParams>();
if (cancelParams == null)
{
continue;
}
_requestRouter.CancelRequest(cancelParams.Id);
continue;
}
var type = _requestProcessIdentifier.Identify(descriptor);
_scheduler.Add(
type,
item.Notification.Method,
DoNotification(descriptor, item.Notification)
);
}
if (item.IsError)
{
// TODO:
_outputHandler.Send(item.Error);
}
}
Func <Task> DoNotification(IHandlerDescriptor descriptor, Notification notification)
{
return(async() => {
try
{
await _requestRouter.RouteNotification(descriptor, notification, CancellationToken.None);
}
catch (Exception e)
{
_logger.LogCritical(Events.UnhandledNotification, e, "Unhandled exception executing notification {Method}", notification.Method);
// TODO: Should we rethrow or swallow?
// If an exception happens... the whole system could be in a bad state, hence this throwing currently.
throw;
}
});
}
}
internal KRPCServer(IPAddress address, ushort rpcPort, ushort streamPort,
bool oneRPCPerUpdate = false, uint maxTimePerUpdate = 5000, bool adaptiveRateControl = true, bool blockingRecv = true, uint recvTimeout = 1000)
{
rpcTcpServer = new TCPServer ("RPCServer", address, rpcPort);
streamTcpServer = new TCPServer ("StreamServer", address, streamPort);
rpcServer = new RPCServer (rpcTcpServer);
streamServer = new StreamServer (streamTcpServer);
clientScheduler = new RoundRobinScheduler<IClient<Request,Response>> ();
continuations = new List<RequestContinuation> ();
streamRequests = new Dictionary<IClient<byte,StreamMessage>,IList<StreamRequest>> ();
OneRPCPerUpdate = oneRPCPerUpdate;
MaxTimePerUpdate = maxTimePerUpdate;
AdaptiveRateControl = adaptiveRateControl;
BlockingRecv = blockingRecv;
RecvTimeout = recvTimeout;
// Tie events to underlying server
rpcServer.OnStarted += (s, e) => {
if (OnStarted != null)
OnStarted (this, EventArgs.Empty);
};
rpcServer.OnStopped += (s, e) => {
if (OnStopped != null)
OnStopped (this, EventArgs.Empty);
};
rpcServer.OnClientRequestingConnection += (s, e) => {
if (OnClientRequestingConnection != null)
OnClientRequestingConnection (s, e);
};
rpcServer.OnClientConnected += (s, e) => {
if (OnClientConnected != null)
OnClientConnected (s, e);
};
rpcServer.OnClientDisconnected += (s, e) => {
if (OnClientDisconnected != null)
OnClientDisconnected (s, e);
};
// Add/remove clients from the scheduler
rpcServer.OnClientConnected += (s, e) => clientScheduler.Add (e.Client);
rpcServer.OnClientDisconnected += (s, e) => clientScheduler.Remove (e.Client);
// Add/remove clients from the list of stream requests
streamServer.OnClientConnected += (s, e) => streamRequests [e.Client] = new List<StreamRequest> ();
streamServer.OnClientDisconnected += (s, e) => streamRequests.Remove (e.Client);
// Validate stream client identifiers
streamServer.OnClientRequestingConnection += (s, e) => {
if (rpcServer.Clients.Where (c => c.Guid == e.Client.Guid).Any ())
e.Request.Allow ();
else
e.Request.Deny ();
};
}
internal KRPCServer(IPAddress address, ushort rpcPort, ushort streamPort,
bool oneRPCPerUpdate = false, uint maxTimePerUpdate = 5000,
bool adaptiveRateControl = true, bool blockingRecv = true, uint recvTimeout = 1000)
{
rpcTcpServer = new TCPServer("RPCServer", address, rpcPort);
streamTcpServer = new TCPServer("StreamServer", address, streamPort);
rpcServer = new RPCServer(rpcTcpServer);
streamServer = new StreamServer(streamTcpServer);
clientScheduler = new RoundRobinScheduler <IClient <Request, Response> > ();
continuations = new List <RequestContinuation> ();
streamRequests = new Dictionary <IClient <byte, StreamMessage>, IList <StreamRequest> > ();
OneRPCPerUpdate = oneRPCPerUpdate;
MaxTimePerUpdate = maxTimePerUpdate;
AdaptiveRateControl = adaptiveRateControl;
BlockingRecv = blockingRecv;
RecvTimeout = recvTimeout;
// Tie events to underlying server
rpcServer.OnStarted += (s, e) => {
if (OnStarted != null)
{
OnStarted(this, EventArgs.Empty);
}
};
rpcServer.OnStopped += (s, e) => {
if (OnStopped != null)
{
OnStopped(this, EventArgs.Empty);
}
};
rpcServer.OnClientRequestingConnection += (s, e) => {
if (OnClientRequestingConnection != null)
{
OnClientRequestingConnection(s, e);
}
};
rpcServer.OnClientConnected += (s, e) => {
if (OnClientConnected != null)
{
OnClientConnected(s, e);
}
};
rpcServer.OnClientDisconnected += (s, e) => {
if (OnClientDisconnected != null)
{
OnClientDisconnected(s, e);
}
};
// Add/remove clients from the scheduler
rpcServer.OnClientConnected += (s, e) => clientScheduler.Add(e.Client);
rpcServer.OnClientDisconnected += (s, e) => clientScheduler.Remove(e.Client);
// Add/remove clients from the list of stream requests
streamServer.OnClientConnected += (s, e) => streamRequests [e.Client] = new List <StreamRequest> ();
streamServer.OnClientDisconnected += (s, e) => streamRequests.Remove(e.Client);
// Validate stream client identifiers
streamServer.OnClientRequestingConnection += (s, e) => {
if (rpcServer.Clients.Where(c => c.Guid == e.Client.Guid).Any())
{
e.Request.Allow();
}
else
{
e.Request.Deny();
}
};
}
private void HandleRequest(string request)
{
JToken payload;
try
{
payload = JToken.Parse(request);
}
catch
{
_outputHandler.Send(new ParseError());
return;
}
if (!_reciever.IsValid(payload))
{
_outputHandler.Send(new InvalidRequest());
return;
}
var(requests, hasResponse) = _reciever.GetRequests(payload);
if (hasResponse)
{
foreach (var response in requests.Where(x => x.IsResponse).Select(x => x.Response))
{
var id = response.Id is string s?long.Parse(s) : response.Id is long l ? l : -1;
if (id < 0)
{
continue;
}
var tcs = _responseRouter.GetRequest(id);
if (tcs is null)
{
continue;
}
if (response is ServerResponse serverResponse)
{
tcs.SetResult(serverResponse.Result);
}
else if (response is ServerError serverError)
{
tcs.SetException(new Exception(JsonConvert.SerializeObject(serverError.Error)));
}
}
return;
}
foreach (var item in requests)
{
if (item.IsRequest)
{
var descriptor = _requestRouter.GetDescriptor(item.Request);
if (descriptor is null)
{
continue;
}
var type = _requestProcessIdentifier.Identify(descriptor);
_scheduler.Add(
type,
item.Request.Method,
async() => {
try
{
var result = await _requestRouter.RouteRequest(descriptor, item.Request);
_outputHandler.Send(result.Value);
}
catch (Exception e)
{
_logger.LogCritical(Events.UnhandledRequest, e, "Unhandled exception executing request {Method}@{Id}", item.Request.Method, item.Request.Id);
// TODO: Should we rethrow or swallow?
// If an exception happens... the whole system could be in a bad state, hence this throwing currently.
throw;
}
}
);
}
else if (item.IsNotification)
{
var descriptor = _requestRouter.GetDescriptor(item.Notification);
if (descriptor is null)
{
continue;
}
var type = _requestProcessIdentifier.Identify(descriptor);
_scheduler.Add(
type,
item.Notification.Method,
async() => {
try
{
await _requestRouter.RouteNotification(descriptor, item.Notification);
}
catch (Exception e)
{
_logger.LogCritical(Events.UnhandledNotification, e, "Unhandled exception executing notification {Method}", item.Notification.Method);
// TODO: Should we rethrow or swallow?
// If an exception happens... the whole system could be in a bad state, hence this throwing currently.
throw;
}
}
);
}
else if (item.IsError)
{
// TODO:
_outputHandler.Send(item.Error);
}
}
}
public static void Add(this IScheduler scheduler, RequestProcessType type, string name, Func <Task> request)
{
scheduler.Add(type, name, Observable.FromAsync(request));
}