private void Run()
{
Logger.Info("进入主线程");
CurStartTime = DateTime.Now;
Schduler.Reset();
BeforeCrawl?.Invoke();
Schduler.AddScanUrl(Config.ScanUrls);
ReadMysqlConfigFromFile();
}
public void Crawl(long maxStep, QueryType queryType)
{
long currentStep = 0;
using (ICheckinEntryProvider checkinEntryProvider = new CheckinEntryProvider())
using (IAllCheckinStorageProvider storageProvider = new AllCheckinSqlStorageProvider())
{
checkinEntryProvider.Connect();
while (currentStep < maxStep && !keywordSequence.EndOfSequence)
{
var keyword = keywordSequence.Current;
try
{
if (BeforeCrawl != null)
{
var args = new BeforeCrawlEventArgs
{
Cancel = false,
Progress = new CrawlProgress
{
Current = currentStep,
Total = maxStep,
CurrentKeyword = keyword,
Message = "Already processed. Skip!",
}
};
BeforeCrawl.Invoke(this, args);
if (args.Cancel)
{
continue;
}
}
var entries = checkinEntryProvider.GetEntries(queryType, keyword).ToList();
foreach (var entry in entries)
{
if (entry.Birthdate != null)
{
entry.Birthdate = storageProvider.NormalizeDateTime(entry.Birthdate.Value);
}
if (entry.CheckinTime != null)
{
entry.CheckinTime = storageProvider.NormalizeDateTime(entry.CheckinTime.Value);
}
storageProvider.SaveEntry(entry);
}
if (AfterCrawl != null)
{
AfterCrawl.Invoke(this, new AfterCrawlEventArgs
{
Progress = new CrawlProgress
{
Current = currentStep,
Total = maxStep,
CurrentKeyword = keyword,
Message = string.Format("{0,5} entry(s) updated.", entries.Count)
}
});
}
}
catch (Exception e)
{
Trace.TraceError("[keyword = {0}] {1}", keyword, e);
if (AfterCrawl != null)
{
AfterCrawl.Invoke(this, new AfterCrawlEventArgs
{
Error = e,
Progress = new CrawlProgress
{
Current = currentStep,
Total = maxStep,
CurrentKeyword = keyword,
Message = "Error!"
}
});
}
}
finally
{
++currentStep;
keywordSequence.MoveNext();
Thread.Sleep(PauseInterval);
}
}
}
}
