Esempio n. 1
0
        public void FrameworkMessage(ISchedulerDriver driver, ExecutorID executorId, SlaveID slaveId, byte[] data)
        {
            var message = JsonHelper.Deserialize<Message> (data);
            switch (message.Type) {
            case "CrawlResult":
                var crawlResult = JsonHelper.Deserialize<CrawlResultMessage> (message.Body);
                Console.WriteLine ($"Framework message <CrawlResult>: got {crawlResult.Links.Length} links from url '{crawlResult.Url}'.");

                foreach (var link in crawlResult.Links) {
                    if (_crawled.Contains (link))
                        continue;

                    _crawlQueue.Enqueue (link);
                    _renderQueue.Enqueue (link);
                }

                // update edges: url -> links
                var edges = _crawlResults.GetOrAdd (crawlResult.Url, x => new List<string> ());
                edges.AddRange (crawlResult.Links);

                // empty edge list for links
                foreach (var link in crawlResult.Links)
                    _crawlResults.GetOrAdd (link, x => new List<string> ());
                break;
            case "RenderResult":
                var renderResult = JsonHelper.Deserialize<RenderResultMessage> (message.Body);
                Console.WriteLine ($"Framework message <RenderResult>: saved '{renderResult.FileName}' for url '{renderResult.Url}'.");

                _renderResults [renderResult.Url] = renderResult.FileName;
                break;
            default:
                Console.WriteLine ($"Unrecognized message type: '{message.Type}'");
                break;
            }
        }
Esempio n. 2
0
 public Status SendFrameworkMessage(ExecutorID executorId, SlaveID slaveId, byte[] data)
 {
     return _bridge.SendFrameworkMessage(executorId, slaveId, data);
 }
Esempio n. 3
0
 public void ExecutorLost(ISchedulerDriver driver, ExecutorID executorId, SlaveID slaveId, int status)
 {
 }
Esempio n. 4
0
 public void SlaveLost(ISchedulerDriver driver, SlaveID slaveId)
 {
 }