public async Task ProcessGallery(string folderName, Action <byte[], int> imageProcessor)
{
for (int index = StartIndex; index <= EndIndex; index++)
{
var idLocation = StartUrl.LastIndexOf(StartIndex.ToString());
var indexLength = StartIndex.ToString().Length;
var url = $"{StartUrl.Substring(0, idLocation)}{index}{StartUrl.Substring(idLocation + indexLength)}";
var wc = new WebClient();
var data = await wc.DownloadDataTaskAsync(url);
imageProcessor(data, index - StartIndex);
}
}
public async Task run()
{
Console.WriteLine($"\n初始网址: {StartUrl}\n");
Console.WriteLine("文件类型:");
foreach (var fileType in FileTypes)
{
Console.WriteLine($" {fileType}");
}
PendingUrl.Enqueue(StartUrl.ToString());
while (PendingUrl.Count > 0)
{
await handle(PendingUrl.Dequeue());
}
}
protected SpiderSetting(SerializationInfo info, StreamingContext context)
{
this.name = info.GetString("name");
this.startUrl = info.GetValue("startUrl", typeof(StartUrl)) as StartUrl;
this.maxDepth = info.GetInt16("maxDepth");
this.allowRedirect = info.GetBoolean("allowRedirect");
this.requestTimeout = info.GetInt32("requestTimeout");
this.iOTimeout = info.GetInt32("iOTimeout");
this.readBufferSize = info.GetInt32("readBufferSize");
this.crawlThreads = info.GetInt16("crawlThreads");
this.processThreads = info.GetInt16("processThreads");
this.proxy = info.GetValue("proxy", typeof(IWebProxy)) as IWebProxy;
this.userAgent = info.GetString("userAgent");
this.referer = info.GetString("referer");
this.cookies = info.GetValue("cookies", typeof(CookieCollection)) as CookieCollection;
this.requestEncoding = info.GetValue("requestEncoding", typeof(Encoding)) as Encoding;
this.urlExtractor = info.GetValue("urlExtractor", typeof(UrlExtractor)) as UrlExtractor;
this.contentHandlers = info.GetValue("contentHandlers", typeof(ContentHandlerCollection)) as ContentHandlerCollection;
this.logger = info.GetValue("logger", typeof(ILogger)) as ILogger;
this.memLimitSize = info.GetInt32("memLimitSize");
this.depositePath = info.GetString("depositePath");
this.speedMode = (SpeedModes)info.GetValue("speedMode", typeof(SpeedModes));
}
public bool HasStartPage()
{
return(!StartUrl.IsNullOrEmpty());
}