public void NewTask(string fullClassName, SpiderSettings settings, SpiderCallback callback) { Type tp = assembly.GetType(fullClassName); Spider spider = (Spider)Activator.CreateInstance(tp); spider.NewTask(settings, callback); }
public void NewTask(SpiderSettings settings, SpiderCallback callback) { DateTime beforWorkDt = DateTime.Now; this.spiderCallback = callback; // 开始任务工作 try { ImportSettings(settings); // 导入配置 BeginWork(); } catch (ThreadAbortException) { // 进程正在被中止 // 不进行操作 } catch (Exception e) { // 任务执行中抛出的错误被接住了... LogError(e.Message); Logging.Error(e.ToString()); // 保存错误详情 } // 任务执行完毕 DateTime afterWorkDt = DateTime.Now; double timeSpent = afterWorkDt.Subtract(beforWorkDt).TotalSeconds; Log("\n"); Log($">> 任务执行完毕 (执行耗时:{timeSpent.ToString()}s)"); RunJs($"Task.get('{settings.TaskId}').taskIsEnd();"); // 报告JS任务结束 Utils.ReleaseMemory(true); }
public TaskRunner(MainForm form, SpiderCallback spiderCallback) { _form = form; _spiderCallback = spiderCallback; }
// Public methods. /// <summary> /// Begins an asynchronous spider crawling using the specified user state. /// </summary> /// <param name="callback"></param> /// <param name="userState">The user state.</param> /// <returns>The result of the asynchronous spider operation.</returns> public IAsyncResult BeginCrawl(SpiderCallback callback, object userState = null) { // Update the spider state. base.OnStarted(); try { // Compute the standard feeds to crawl. Dictionary<string, DbObjectStandardFeed> feeds = new Dictionary<string, DbObjectStandardFeed>(); // For all standard feeds. foreach (YouTubeStandardFeed feed in InetApi.YouTube.Api.V2.YouTube.StandardFeeds) { // If the feed is not selected, continue. if (!this.GetFeedSelected(feed)) continue; // Get the valid times for this feed. YouTubeTimeId[] times = YouTubeUri.GetValidTime(feed); // For all times corresponding to this feed. foreach (YouTubeTimeId time in times) { // Create a new standard feed object. DbObjectStandardFeed obj = new DbObjectStandardFeed(); obj.Id = this.EncodeFeedKey(feed, time, null, null); obj.FeedId = (int)feed; obj.TimeId = (int)time; obj.Category = null; obj.Region = null; feeds.Add(obj.Id, obj); // For all assignable and non-deprecated categories. foreach (YouTubeCategory category in this.crawler.YouTube.Categories) { // If the category supports browsable regions. if (category.Browsable != null) { // Create a new standard feed object. obj = new DbObjectStandardFeed(); obj.Id = this.EncodeFeedKey(feed, time, category.Label, null); obj.FeedId = (int)feed; obj.TimeId = (int)time; obj.Category = category.Term; obj.Region = null; feeds.Add(obj.Id, obj); // For all browsable regions. foreach (string region in category.Browsable) { // Create a new standard feed object. obj = new DbObjectStandardFeed(); obj.Id = this.EncodeFeedKey(feed, time, category.Label, region); obj.FeedId = (int)feed; obj.TimeId = (int)time; obj.Category = category.Label; obj.Region = region; feeds.Add(obj.Id, obj); } } } } } // Raise the crawl feeds started event. if (this.FeedsCrawlStarted != null) this.FeedsCrawlStarted(this, new SpiderInfoEventArgs<CrawlInfo>(this, new CrawlInfo(feeds))); // Create a new spider asynchronous result. SpiderAsyncResult asyncResult = new SpiderAsyncResult(userState); // Set the crawl result counters. int counterSuccess = 0; int counterWarning = 0; int counterFailed = 0; int counterPending = feeds.Count; // Execute the crawl on the thread pool. ThreadPool.QueueUserWorkItem((object state) => { // Set the feed index. int index = 0; // For each feed in the feeds collection. foreach(KeyValuePair<string, DbObjectStandardFeed> feed in feeds) { // Check if the crawl has been canceled. if (asyncResult.IsCanceled) break; // Increment the feed index. index++; // Get the object. DbObjectStandardFeed obj = feed.Value; // Call the feed started event handler. if (this.FeedCrawlStarted != null) this.FeedCrawlStarted(this, new SpiderInfoEventArgs<FeedStartedInfo>(this, new FeedStartedInfo(obj, index, feeds.Count))); // Crawl the feed. CrawlResult result = this.CrawlFeed( asyncResult, (YouTubeStandardFeed)obj.FeedId, (YouTubeTimeId)obj.TimeId, obj.Category, obj.Region, ref obj); // Call the feed finished event handler. if (this.FeedCrawlFinished != null) this.FeedCrawlFinished(this, new SpiderInfoEventArgs<FeedFinishedInfo>(this, new FeedFinishedInfo(obj, index, feeds.Count, result))); } // Set the result. asyncResult.Result = feeds; // Raise the crawl feeds finished event. if (this.FeedsCrawlFinished != null) this.FeedsCrawlFinished(this, new SpiderInfoEventArgs<CrawlInfo>(this, new CrawlInfo(feeds))); // Update the spider state. base.OnFinished(); }); // Returns the spider object as the asynchronous state. return asyncResult; } catch (Exception) { // If an exception occurs, update the spider state. base.OnFinished(); // Rethrow the exception. throw; } }