/// <inheritdoc/> public async Task RegisterAsync(IConsulRegistration registration) { if (registration == null) { throw new ArgumentNullException(nameof(registration)); } _logger?.LogInformation("Registering service with consul {serviceId} ", registration.ServiceId); try { await _client.Agent.ServiceRegister(registration.Service); if (Options.IsHeartBeatEnabled && _scheduler != null) { _scheduler.Add(registration.InstanceId); } } catch (Exception e) { if (Options.FailFast) { _logger?.LogError(e, "Error registering service with consul {serviceId} ", registration.ServiceId); throw; } _logger?.LogWarning(e, "Failfast is false. Error registering service with consul {serviceId} ", registration.ServiceId); } }
protected ScheduledService(ISchedule schedule, IScheduler scheduler) { Scheduler = scheduler; Schedule = schedule; scheduler.Add(schedule, Run); }
public IEnumerable <IVertex> Crawl(Uri link) { //Parse first page var exploredVertex = _parser.Parse(link); _scheduler.Add(exploredVertex); ////Next scheduled pages while (_scheduler.Any()) { foreach (var unexploredVertex in _scheduler) { var friends = _parser.ParseFriends(unexploredVertex); _scheduler.Add(friends); _scheduler.Remove(unexploredVertex); yield return(unexploredVertex); } } }
public async Task <CrawlResult> Crawl(Uri uri, CancellationTokenSource cancellationTokenSource) { if (uri == null) { throw new ArgumentNullException(nameof(uri)); } _crawlContext.OriginalRootUri = uri; if (cancellationTokenSource != null) { _crawlContext.CancellationTokenSource = cancellationTokenSource; } _crawlResult = new CrawlResult { RootUri = _crawlContext.OriginalRootUri, CrawlContext = _crawlContext }; var timer = Stopwatch.StartNew(); try { var rootPage = new PageToCrawl(uri) { ParentUri = uri, IsInternal = true, IsRoot = true }; _scheduler.Add(rootPage); await CrawlSite().ConfigureAwait(false); } catch (Exception e) { _crawlResult.ErrorException = e; } finally { _threadManager?.Dispose(); } timer.Stop(); _crawlResult.Elapsed = timer.Elapsed; return(_crawlResult); }
private void _trayIcon_MoreOptionsOnClick(object sender, MoreOptionsOnClickEventArgs e) { var dialog = new MoreOptionsDialog(_translation); var result = dialog.ShowDialog(e.Socket); if (!result.Canceled) { _scheduler.Add(new ScheduledTask( DateTime.Now + result.TimeSpan, result.Status, result.Socket)); } }
static void Main(string[] args) { IScheduler scheduler = SchedulerFactory.Create(SchedulerType.RoundRobin); foreach (string s in Directory.GetFiles(@"C:\Downloads")) { scheduler.Add(new CopyFileAsync(s, @"C:\Downloads\COPYTEST\COPYDEST")); } scheduler.CoroutineComplete += new CoroutineCompleteHandler(scheduler_CoroutineComplete); scheduler.Run(); Console.ReadKey(); }
public async Task Crawl(Uri startingUrl, int maxUrlsToCrawl) { var maxCrawls = maxUrlsToCrawl; var currentCrawls = 0; var crawlComplete = false; _scheduler.Add(startingUrl); while (!crawlComplete && currentCrawls < maxCrawls) { var linksLeftToSchedule = _scheduler.Count; if (linksLeftToSchedule > 0) { var linkToCrawl = _scheduler.GetNext(); ProcessPage(await _pageRequester.MakeRequest(linkToCrawl)); currentCrawls++; } else { //Finished the crawl crawlComplete = true; } } }
internal KRPCServer(IPAddress address, ushort rpcPort, ushort streamPort) { rpcTcpServer = new TCPServer ("RPCServer", address, rpcPort); streamTcpServer = new TCPServer ("StreamServer", address, streamPort); rpcServer = new RPCServer (rpcTcpServer); streamServer = new StreamServer (streamTcpServer); clientScheduler = new RoundRobinScheduler<IClient<Request,Response>> (); continuations = new List<RequestContinuation> (); streamRequests = new Dictionary<IClient<byte,StreamMessage>,IList<StreamRequest>> (); // Tie events to underlying server rpcServer.OnStarted += (s, e) => { if (OnStarted != null) OnStarted (this, EventArgs.Empty); }; rpcServer.OnStopped += (s, e) => { if (OnStopped != null) OnStopped (this, EventArgs.Empty); }; rpcServer.OnClientRequestingConnection += (s, e) => { if (OnClientRequestingConnection != null) OnClientRequestingConnection (s, e); }; rpcServer.OnClientConnected += (s, e) => { if (OnClientConnected != null) OnClientConnected (s, e); }; rpcServer.OnClientDisconnected += (s, e) => { if (OnClientDisconnected != null) OnClientDisconnected (s, e); }; // Add/remove clients from the scheduler rpcServer.OnClientConnected += (s, e) => clientScheduler.Add (e.Client); rpcServer.OnClientDisconnected += (s, e) => clientScheduler.Remove (e.Client); // Add/remove clients from the list of stream requests streamServer.OnClientConnected += (s, e) => streamRequests [e.Client] = new List<StreamRequest> (); streamServer.OnClientDisconnected += (s, e) => streamRequests.Remove (e.Client); // Validate stream client identifiers streamServer.OnClientRequestingConnection += (s, e) => { if (rpcServer.Clients.Where (c => c.Guid == e.Client.Guid).Any ()) e.Request.Allow (); else e.Request.Deny (); }; }
public void EnabledTest() { TimeSpan triggerPeriod = new TimeSpan(5, 0, 0); ISchedule target = theScheduler.Add(triggerPeriod, false); Assert.IsFalse(target.Enabled); target.Enabled = true; DateTime nextDueTime = DateTime.Now + target.Period; Assert.AreNotEqual(target.NextDueTime, DateTime.MaxValue); Assert.AreEqual(target.NextDueTime, nextDueTime); target.Enabled = false; Assert.AreEqual(target.NextDueTime, DateTime.MaxValue); }
public void AddTest() { ISchedule createdSchedule = theScheduler.Add(new TimeSpan(0, 0, 5), true); Assert.IsTrue(theScheduler.Contains(createdSchedule), "Created schedule couldn't be found"); Console.WriteLine("Next time schedule is signaled: " + createdSchedule.NextDueTime); int waitTimeInMilliseconds = Convert.ToInt32(createdSchedule.Period.TotalMilliseconds); Console.WriteLine("Waiting for: " + waitTimeInMilliseconds + " ms to be signaled."); if (createdSchedule.ScheduleSignal.WaitOne(waitTimeInMilliseconds * 2, false)) { Console.WriteLine("Signal was set in expected wait interval."); } else { Assert.Fail("Signal was not set in expected wait interval."); } //We add a new Schedule and look if the timing mechanism is recalculated ISchedule fastSchedule = theScheduler.Add(new TimeSpan(0, 0, 1), true); Assert.IsTrue(theScheduler.Contains(createdSchedule), "Created schedule couldn't be found"); if (fastSchedule.ScheduleSignal.WaitOne(1000 * 2, false)) { Console.WriteLine("Fast signal was set in expected wait interval."); } else { Assert.Fail("Fast signal was not set in expected wait interval."); } theScheduler.Remove(createdSchedule); theScheduler.Remove(fastSchedule); }
private async Task RegisterAsyncInternal(IConsulRegistration registration) { _logger?.LogInformation("Registering service with consul {serviceId} ", registration.ServiceId); try { await _client.Agent.ServiceRegister(registration.Service).ConfigureAwait(false); if (Options.IsHeartBeatEnabled && _scheduler != null) { _scheduler.Add(registration.InstanceId); } } catch (Exception e) { if (Options.FailFast) { _logger?.LogError(e, "Error registering service with consul {serviceId} ", registration.ServiceId); throw; } _logger?.LogWarning(e, "Failfast is false. Error registering service with consul {serviceId} ", registration.ServiceId); } }
/// <summary> /// Begins a synchronous crawl using the uri param, subscribe to events to process data as it becomes available /// </summary> public virtual async Task <CrawlResult> CrawlAsync(Uri uri, CancellationTokenSource cancellationTokenSource) { if (uri == null) { throw new ArgumentNullException(nameof(uri)); } _crawlContext.RootUri = _crawlContext.OriginalRootUri = uri; if (cancellationTokenSource != null) { _crawlContext.CancellationTokenSource = cancellationTokenSource; } _crawlResult = new CrawlResult(); _crawlResult.RootUri = _crawlContext.RootUri; _crawlResult.CrawlContext = _crawlContext; _crawlComplete = false; _logger.LogInformation($"About to crawl site [{uri.AbsoluteUri}]"); PrintConfigValues(_crawlContext.CrawlConfiguration); if (_memoryManager != null) { _crawlContext.MemoryUsageBeforeCrawlInMb = _memoryManager.GetCurrentUsageInMb(); _logger.LogInformation($"Starting memory usage for site [{uri.AbsoluteUri}] is [{_crawlContext.MemoryUsageBeforeCrawlInMb}mb]"); } _crawlContext.CrawlStartDate = DateTime.Now; var timer = Stopwatch.StartNew(); if (_crawlContext.CrawlConfiguration.CrawlTimeoutSeconds > 0) { _timeoutTimer = new Timer(HandleCrawlTimeout, null, 0, _crawlContext.CrawlConfiguration.CrawlTimeoutSeconds * 1000); } try { var rootPage = new PageToCrawl(uri) { ParentUri = uri, IsInternal = true, IsRoot = true }; if (ShouldSchedulePageLink(rootPage)) { _scheduler.Add(rootPage); } VerifyRequiredAvailableMemory(); await CrawlSite(); } catch (Exception e) { _crawlResult.ErrorException = e; _logger.LogCritical($"An error occurred while crawling site [{uri}]", e); } finally { _threadManager?.Dispose(); } _timeoutTimer?.Dispose(); timer.Stop(); if (_memoryManager != null) { _crawlContext.MemoryUsageAfterCrawlInMb = _memoryManager.GetCurrentUsageInMb(); _logger.LogInformation($"Ending memory usage for site [{uri.AbsoluteUri}] is [{_crawlContext.MemoryUsageAfterCrawlInMb}mb]"); } _crawlResult.Elapsed = timer.Elapsed; _logger.LogInformation($"Crawl complete for site [{_crawlResult.RootUri.AbsoluteUri}]: Crawled [{_crawlResult.CrawlContext.CrawledCount}] pages in [{_crawlResult.Elapsed}]"); return(_crawlResult); }
/// <summary> /// Begins a synchronous crawl using the uri param, subscribe to events to process data as it becomes available /// </summary> public virtual CrawlResult Crawl(Uri uri, CancellationTokenSource cancellationTokenSource) { if (uri == null) { throw new ArgumentNullException("uri"); } _crawlContext.RootUri = uri; if (cancellationTokenSource != null) { _crawlContext.CancellationTokenSource = cancellationTokenSource; } _crawlResult = new CrawlResult(); _crawlResult.RootUri = _crawlContext.RootUri; _crawlResult.CrawlContext = _crawlContext; _crawlComplete = false; _logger.InfoFormat("About to crawl site [{0}]", uri.AbsoluteUri); PrintConfigValues(_crawlContext.CrawlConfiguration); if (_memoryManager != null) { _crawlContext.MemoryUsageBeforeCrawlInMb = _memoryManager.GetCurrentUsageInMb(); _logger.InfoFormat("Starting memory usage for site [{0}] is [{1}mb]", uri.AbsoluteUri, _crawlContext.MemoryUsageBeforeCrawlInMb); } _crawlContext.CrawlStartDate = DateTime.Now; Stopwatch timer = Stopwatch.StartNew(); if (_crawlContext.CrawlConfiguration.CrawlTimeoutSeconds > 0) { _timeoutTimer = new Timer(_crawlContext.CrawlConfiguration.CrawlTimeoutSeconds * 1000); _timeoutTimer.Elapsed += HandleCrawlTimeout; _timeoutTimer.Start(); } try { PageToCrawl rootPage = new PageToCrawl(uri) { ParentUri = uri, IsInternal = true, IsRoot = true }; if (ShouldSchedulePageLink(rootPage)) { _scheduler.Add(rootPage); } VerifyRequiredAvailableMemory(); CrawlSite(); } catch (Exception e) { _crawlResult.ErrorException = e; _logger.FatalFormat("An error occurred while crawling site [{0}]", uri); _logger.Fatal(e); } finally { if (_threadManager != null) { _threadManager.Dispose(); } } if (_timeoutTimer != null) { _timeoutTimer.Stop(); } timer.Stop(); if (_memoryManager != null) { _crawlContext.MemoryUsageAfterCrawlInMb = _memoryManager.GetCurrentUsageInMb(); _logger.InfoFormat("Ending memory usage for site [{0}] is [{1}mb]", uri.AbsoluteUri, _crawlContext.MemoryUsageAfterCrawlInMb); } _crawlResult.Elapsed = timer.Elapsed; _logger.InfoFormat("Crawl complete for site [{0}]: Crawled [{1}] pages in [{2}]", _crawlResult.RootUri.AbsoluteUri, _crawlResult.CrawlContext.CrawledCount, _crawlResult.Elapsed); return(_crawlResult); }
private void HandleRequest(string request) { JToken payload; try { payload = JToken.Parse(request); } catch { _outputHandler.Send(new ParseError()); return; } if (!_receiver.IsValid(payload)) { _outputHandler.Send(new InvalidRequest()); return; } var(requests, hasResponse) = _receiver.GetRequests(payload); if (hasResponse) { foreach (var response in requests.Where(x => x.IsResponse).Select(x => x.Response)) { var id = response.Id is string s?long.Parse(s) : response.Id is long l ? l : -1; if (id < 0) { continue; } var tcs = _responseRouter.GetRequest(id); if (tcs is null) { continue; } if (response is ServerResponse serverResponse) { tcs.SetResult(serverResponse.Result); } else if (response is ServerError serverError) { tcs.SetException(new JsonRpcException(serverError)); } } return; } foreach (var item in requests) { if (item.IsRequest) { var descriptor = _requestRouter.GetDescriptor(item.Request); if (descriptor is null) { continue; } var type = _requestProcessIdentifier.Identify(descriptor); _requestRouter.StartRequest(item.Request.Id); _scheduler.Add( type, item.Request.Method, async() => { try { var result = await _requestRouter.RouteRequest(descriptor, item.Request, CancellationToken.None); if (result.IsError && result.Error is RequestCancelled) { return; } _outputHandler.Send(result.Value); } catch (Exception e) { _logger.LogCritical(Events.UnhandledRequest, e, "Unhandled exception executing request {Method}@{Id}", item.Request.Method, item.Request.Id); // TODO: Should we rethrow or swallow? // If an exception happens... the whole system could be in a bad state, hence this throwing currently. throw; } } ); } if (item.IsNotification) { var descriptor = _requestRouter.GetDescriptor(item.Notification); if (descriptor is null) { continue; } // We need to special case cancellation so that we can cancel any request that is currently in flight. if (descriptor.Method == JsonRpcNames.CancelRequest) { var cancelParams = item.Notification.Params?.ToObject <CancelParams>(); if (cancelParams == null) { continue; } _requestRouter.CancelRequest(cancelParams.Id); continue; } var type = _requestProcessIdentifier.Identify(descriptor); _scheduler.Add( type, item.Notification.Method, DoNotification(descriptor, item.Notification) ); } if (item.IsError) { // TODO: _outputHandler.Send(item.Error); } } Func <Task> DoNotification(IHandlerDescriptor descriptor, Notification notification) { return(async() => { try { await _requestRouter.RouteNotification(descriptor, notification, CancellationToken.None); } catch (Exception e) { _logger.LogCritical(Events.UnhandledNotification, e, "Unhandled exception executing notification {Method}", notification.Method); // TODO: Should we rethrow or swallow? // If an exception happens... the whole system could be in a bad state, hence this throwing currently. throw; } }); } }
internal KRPCServer(IPAddress address, ushort rpcPort, ushort streamPort, bool oneRPCPerUpdate = false, uint maxTimePerUpdate = 5000, bool adaptiveRateControl = true, bool blockingRecv = true, uint recvTimeout = 1000) { rpcTcpServer = new TCPServer ("RPCServer", address, rpcPort); streamTcpServer = new TCPServer ("StreamServer", address, streamPort); rpcServer = new RPCServer (rpcTcpServer); streamServer = new StreamServer (streamTcpServer); clientScheduler = new RoundRobinScheduler<IClient<Request,Response>> (); continuations = new List<RequestContinuation> (); streamRequests = new Dictionary<IClient<byte,StreamMessage>,IList<StreamRequest>> (); OneRPCPerUpdate = oneRPCPerUpdate; MaxTimePerUpdate = maxTimePerUpdate; AdaptiveRateControl = adaptiveRateControl; BlockingRecv = blockingRecv; RecvTimeout = recvTimeout; // Tie events to underlying server rpcServer.OnStarted += (s, e) => { if (OnStarted != null) OnStarted (this, EventArgs.Empty); }; rpcServer.OnStopped += (s, e) => { if (OnStopped != null) OnStopped (this, EventArgs.Empty); }; rpcServer.OnClientRequestingConnection += (s, e) => { if (OnClientRequestingConnection != null) OnClientRequestingConnection (s, e); }; rpcServer.OnClientConnected += (s, e) => { if (OnClientConnected != null) OnClientConnected (s, e); }; rpcServer.OnClientDisconnected += (s, e) => { if (OnClientDisconnected != null) OnClientDisconnected (s, e); }; // Add/remove clients from the scheduler rpcServer.OnClientConnected += (s, e) => clientScheduler.Add (e.Client); rpcServer.OnClientDisconnected += (s, e) => clientScheduler.Remove (e.Client); // Add/remove clients from the list of stream requests streamServer.OnClientConnected += (s, e) => streamRequests [e.Client] = new List<StreamRequest> (); streamServer.OnClientDisconnected += (s, e) => streamRequests.Remove (e.Client); // Validate stream client identifiers streamServer.OnClientRequestingConnection += (s, e) => { if (rpcServer.Clients.Where (c => c.Guid == e.Client.Guid).Any ()) e.Request.Allow (); else e.Request.Deny (); }; }
internal KRPCServer(IPAddress address, ushort rpcPort, ushort streamPort, bool oneRPCPerUpdate = false, uint maxTimePerUpdate = 5000, bool adaptiveRateControl = true, bool blockingRecv = true, uint recvTimeout = 1000) { rpcTcpServer = new TCPServer("RPCServer", address, rpcPort); streamTcpServer = new TCPServer("StreamServer", address, streamPort); rpcServer = new RPCServer(rpcTcpServer); streamServer = new StreamServer(streamTcpServer); clientScheduler = new RoundRobinScheduler <IClient <Request, Response> > (); continuations = new List <RequestContinuation> (); streamRequests = new Dictionary <IClient <byte, StreamMessage>, IList <StreamRequest> > (); OneRPCPerUpdate = oneRPCPerUpdate; MaxTimePerUpdate = maxTimePerUpdate; AdaptiveRateControl = adaptiveRateControl; BlockingRecv = blockingRecv; RecvTimeout = recvTimeout; // Tie events to underlying server rpcServer.OnStarted += (s, e) => { if (OnStarted != null) { OnStarted(this, EventArgs.Empty); } }; rpcServer.OnStopped += (s, e) => { if (OnStopped != null) { OnStopped(this, EventArgs.Empty); } }; rpcServer.OnClientRequestingConnection += (s, e) => { if (OnClientRequestingConnection != null) { OnClientRequestingConnection(s, e); } }; rpcServer.OnClientConnected += (s, e) => { if (OnClientConnected != null) { OnClientConnected(s, e); } }; rpcServer.OnClientDisconnected += (s, e) => { if (OnClientDisconnected != null) { OnClientDisconnected(s, e); } }; // Add/remove clients from the scheduler rpcServer.OnClientConnected += (s, e) => clientScheduler.Add(e.Client); rpcServer.OnClientDisconnected += (s, e) => clientScheduler.Remove(e.Client); // Add/remove clients from the list of stream requests streamServer.OnClientConnected += (s, e) => streamRequests [e.Client] = new List <StreamRequest> (); streamServer.OnClientDisconnected += (s, e) => streamRequests.Remove(e.Client); // Validate stream client identifiers streamServer.OnClientRequestingConnection += (s, e) => { if (rpcServer.Clients.Where(c => c.Guid == e.Client.Guid).Any()) { e.Request.Allow(); } else { e.Request.Deny(); } }; }
private void HandleRequest(string request) { JToken payload; try { payload = JToken.Parse(request); } catch { _outputHandler.Send(new ParseError()); return; } if (!_reciever.IsValid(payload)) { _outputHandler.Send(new InvalidRequest()); return; } var(requests, hasResponse) = _reciever.GetRequests(payload); if (hasResponse) { foreach (var response in requests.Where(x => x.IsResponse).Select(x => x.Response)) { var id = response.Id is string s?long.Parse(s) : response.Id is long l ? l : -1; if (id < 0) { continue; } var tcs = _responseRouter.GetRequest(id); if (tcs is null) { continue; } if (response is ServerResponse serverResponse) { tcs.SetResult(serverResponse.Result); } else if (response is ServerError serverError) { tcs.SetException(new Exception(JsonConvert.SerializeObject(serverError.Error))); } } return; } foreach (var item in requests) { if (item.IsRequest) { var descriptor = _requestRouter.GetDescriptor(item.Request); if (descriptor is null) { continue; } var type = _requestProcessIdentifier.Identify(descriptor); _scheduler.Add( type, item.Request.Method, async() => { try { var result = await _requestRouter.RouteRequest(descriptor, item.Request); _outputHandler.Send(result.Value); } catch (Exception e) { _logger.LogCritical(Events.UnhandledRequest, e, "Unhandled exception executing request {Method}@{Id}", item.Request.Method, item.Request.Id); // TODO: Should we rethrow or swallow? // If an exception happens... the whole system could be in a bad state, hence this throwing currently. throw; } } ); } else if (item.IsNotification) { var descriptor = _requestRouter.GetDescriptor(item.Notification); if (descriptor is null) { continue; } var type = _requestProcessIdentifier.Identify(descriptor); _scheduler.Add( type, item.Notification.Method, async() => { try { await _requestRouter.RouteNotification(descriptor, item.Notification); } catch (Exception e) { _logger.LogCritical(Events.UnhandledNotification, e, "Unhandled exception executing notification {Method}", item.Notification.Method); // TODO: Should we rethrow or swallow? // If an exception happens... the whole system could be in a bad state, hence this throwing currently. throw; } } ); } else if (item.IsError) { // TODO: _outputHandler.Send(item.Error); } } }
public static void Add(this IScheduler scheduler, RequestProcessType type, string name, Func <Task> request) { scheduler.Add(type, name, Observable.FromAsync(request)); }