public void NewTask(string fullClassName, SpiderSettings settings, SpiderCallback callback)
{
Type tp = assembly.GetType(fullClassName);
Spider spider = (Spider)Activator.CreateInstance(tp);
spider.NewTask(settings, callback);
}
public void NewTask(SpiderSettings settings, SpiderCallback callback)
{
DateTime beforWorkDt = DateTime.Now;
this.spiderCallback = callback;
// 开始任务工作
try
{
ImportSettings(settings); // 导入配置
BeginWork();
}
catch (ThreadAbortException)
{
// 进程正在被中止
// 不进行操作
}
catch (Exception e)
{
// 任务执行中抛出的错误被接住了...
LogError(e.Message);
Logging.Error(e.ToString()); // 保存错误详情
}
// 任务执行完毕
DateTime afterWorkDt = DateTime.Now;
double timeSpent = afterWorkDt.Subtract(beforWorkDt).TotalSeconds;
Log("\n");
Log($">> 任务执行完毕 （执行耗时：{timeSpent.ToString()}s）");
RunJs($"Task.get('{settings.TaskId}').taskIsEnd();"); // 报告JS任务结束
Utils.ReleaseMemory(true);
}
public TaskRunner(MainForm form, SpiderCallback spiderCallback)
{
_form = form;
_spiderCallback = spiderCallback;
}
// Public methods.
/// <summary>
/// Begins an asynchronous spider crawling using the specified user state.
/// </summary>
/// <param name="callback"></param>
/// <param name="userState">The user state.</param>
/// <returns>The result of the asynchronous spider operation.</returns>
public IAsyncResult BeginCrawl(SpiderCallback callback, object userState = null)
{
// Update the spider state.
base.OnStarted();
try
{
// Compute the standard feeds to crawl.
Dictionary<string, DbObjectStandardFeed> feeds = new Dictionary<string, DbObjectStandardFeed>();
// For all standard feeds.
foreach (YouTubeStandardFeed feed in InetApi.YouTube.Api.V2.YouTube.StandardFeeds)
{
// If the feed is not selected, continue.
if (!this.GetFeedSelected(feed)) continue;
// Get the valid times for this feed.
YouTubeTimeId[] times = YouTubeUri.GetValidTime(feed);
// For all times corresponding to this feed.
foreach (YouTubeTimeId time in times)
{
// Create a new standard feed object.
DbObjectStandardFeed obj = new DbObjectStandardFeed();
obj.Id = this.EncodeFeedKey(feed, time, null, null);
obj.FeedId = (int)feed;
obj.TimeId = (int)time;
obj.Category = null;
obj.Region = null;
feeds.Add(obj.Id, obj);
// For all assignable and non-deprecated categories.
foreach (YouTubeCategory category in this.crawler.YouTube.Categories)
{
// If the category supports browsable regions.
if (category.Browsable != null)
{
// Create a new standard feed object.
obj = new DbObjectStandardFeed();
obj.Id = this.EncodeFeedKey(feed, time, category.Label, null);
obj.FeedId = (int)feed;
obj.TimeId = (int)time;
obj.Category = category.Term;
obj.Region = null;
feeds.Add(obj.Id, obj);
// For all browsable regions.
foreach (string region in category.Browsable)
{
// Create a new standard feed object.
obj = new DbObjectStandardFeed();
obj.Id = this.EncodeFeedKey(feed, time, category.Label, region);
obj.FeedId = (int)feed;
obj.TimeId = (int)time;
obj.Category = category.Label;
obj.Region = region;
feeds.Add(obj.Id, obj);
}
}
}
}
}
// Raise the crawl feeds started event.
if (this.FeedsCrawlStarted != null) this.FeedsCrawlStarted(this, new SpiderInfoEventArgs<CrawlInfo>(this, new CrawlInfo(feeds)));
// Create a new spider asynchronous result.
SpiderAsyncResult asyncResult = new SpiderAsyncResult(userState);
// Set the crawl result counters.
int counterSuccess = 0;
int counterWarning = 0;
int counterFailed = 0;
int counterPending = feeds.Count;
// Execute the crawl on the thread pool.
ThreadPool.QueueUserWorkItem((object state) =>
{
// Set the feed index.
int index = 0;
// For each feed in the feeds collection.
foreach(KeyValuePair<string, DbObjectStandardFeed> feed in feeds)
{
// Check if the crawl has been canceled.
if (asyncResult.IsCanceled) break;
// Increment the feed index.
index++;
// Get the object.
DbObjectStandardFeed obj = feed.Value;
// Call the feed started event handler.
if (this.FeedCrawlStarted != null) this.FeedCrawlStarted(this, new SpiderInfoEventArgs<FeedStartedInfo>(this, new FeedStartedInfo(obj, index, feeds.Count)));
// Crawl the feed.
CrawlResult result = this.CrawlFeed(
asyncResult,
(YouTubeStandardFeed)obj.FeedId,
(YouTubeTimeId)obj.TimeId,
obj.Category,
obj.Region,
ref obj);
// Call the feed finished event handler.
if (this.FeedCrawlFinished != null) this.FeedCrawlFinished(this, new SpiderInfoEventArgs<FeedFinishedInfo>(this, new FeedFinishedInfo(obj, index, feeds.Count, result)));
}
// Set the result.
asyncResult.Result = feeds;
// Raise the crawl feeds finished event.
if (this.FeedsCrawlFinished != null) this.FeedsCrawlFinished(this, new SpiderInfoEventArgs<CrawlInfo>(this, new CrawlInfo(feeds)));
// Update the spider state.
base.OnFinished();
});
// Returns the spider object as the asynchronous state.
return asyncResult;
}
catch (Exception)
{
// If an exception occurs, update the spider state.
base.OnFinished();
// Rethrow the exception.
throw;
}
}
