Exemplo n.º 1
0
        public override void Run()
        {
            Trace.TraceInformation("WorkerRole1 is running");
            Storage.CreateStorage();
            while (true)
            {
                CloudQueueMessage command = Storage.CommandQueue.GetMessage(TimeSpan.FromMinutes(5));
                if (command != null)
                {
                    switch (command.AsString)
                    {
                    case "Initialize Crawl":
                        crawler.Start();
                        break;

                    case "Stop Crawl":
                        crawler.Stop();
                        crawler.updateDashboard();
                        break;

                    case "Resume Crawl":
                        crawler.Resume();
                        break;

                    default:
                        break;
                    }
                    Storage.CommandQueue.DeleteMessage(command);
                }
                if (crawler.GetCrawlerState().Equals("Crawling") || crawler.GetCrawlerState().Equals("Loading"))
                {
                    CloudQueueMessage link = Storage.LinkQueue.GetMessage(TimeSpan.FromMinutes(5));
                    if (link != null)
                    {
                        linkCount++;
                        try
                        {
                            crawler.CrawlUrl(link.AsString);
                            Storage.LinkQueue.DeleteMessage(link);
                        }
                        catch { }
                        if (linkCount % 5 == 0)
                        {
                            crawler.updateDashboard();
                        }
                    }
                }
            }
        }
Exemplo n.º 2
0
        public override void Run()
        {
            Trace.TraceInformation("WorkerRole1 is running");


            while (true)
            {
                CloudQueueMessage commandMessage = StorageManager.CommandQueue().GetMessage();
                if (commandMessage != null)
                {
                    switch (commandMessage.AsString)
                    {
                    case "startcrawling":
                        crawler.Start();
                        StorageManager.CommandQueue().DeleteMessage(commandMessage);
                        break;

                    case "stopcrawling":
                        crawler.Stop();
                        StorageManager.CommandQueue().DeleteMessage(commandMessage);
                        break;

                    case "clear":
                        StorageManager.LinkQueue().Clear();
                        StorageManager.CommandQueue().Clear();
                        StorageManager.HTMLQueue().Clear();
                        StorageManager.GetTable().DeleteIfExists();
                        StorageManager.PerformanceCounterTable().DeleteIfExists();
                        StorageManager.ErrorTable().DeleteIfExists();
                        break;

                    default:
                        break;
                    }
                }
                new Task(crawler.GetPerfCounters).Start();
                if (Crawler.state.Equals("Loading"))
                {
                    new Task(crawler.GetPerfCounters).Start();
                    crawler.CrawlUrl();
                }
                else if (crawler.GetState().Equals("Crawling"))
                {
                    new Task(crawler.GetPerfCounters).Start();
                    crawler.GetHTMLData();
                }
            }
        }