private void Crawl()
{
List <Action> actionss = new List <Action>();
while (true)
{
string current = null;
foreach (string url in urls.Keys)
{
if ((bool)urls[url])
{
continue;
}
{
current = url;
}
}
if (current == null || count > 10)
{
break;
}
Crawlers crawlers1 = new Crawlers(current, urls, count);
actionss.Add(crawlers1.crawlers);
urls = crawlers1.urls;
Console.WriteLine("爬行" + current + "页面!");
while (urls.Count == 1)
{
string html = Download(current);
urls[current] = true;
count++;
Parse(html);
}
}
Parallel.Invoke(actionss.ToArray());
Console.WriteLine("爬行结束");
}
private static int CrawlerCount(string x) => Crawlers.Count(y => x.ToLower().Contains(y.ToString().ToLower()));
public override bool RemoveCrawler(IProviderCrawler crawler)
{
return(Crawlers.Remove(crawler));
}
public void AddCrawler(IProviderCrawler crawler)
{
crawler.Initialize(this);
Crawlers.Add(crawler);
}
public JsonResult Upload(string collection = null, string ac = null)
{
IUpload UpOp = WebIoc.Container.Resolve <IUpload>();
var context = ControllerContext.HttpContext;
if (string.IsNullOrEmpty(ac))
{
ac = context.Request["action"];
}
switch (ac)
{
case "config": return(UploadConfig());
case "uploadimage":
{
UploadResult result = UpOp.Upload(new UploadConfig()
{
AllowExtensions = CustomConfig.ImgExts.ToArray(),
PathFormat = CustomConfig.EditorImgPath,
SizeLimit = Convert.ToInt32(CustomConfig.CanUploadImgSize * 1024 * 1024),
UploadFieldName = "upfile"
});
return(Json(new
{
fileid = result.FileId,
state = UpOp.GetStateMessage(result.State),
url = result.Url,
title = result.OriginFileName,
original = result.OriginFileName,
error = result.ErrorMessage
}, "text/html"));
}
case "uploadscrawl":
{
UploadResult result = UpOp.Upload(new UploadConfig()
{
AllowExtensions = CustomConfig.ImgExts.ToArray(),
PathFormat = CustomConfig.EditorImgPath,
SizeLimit = Convert.ToInt32(CustomConfig.CanUploadImgSize * 1024 * 1024),
UploadFieldName = "upfile",
Base64 = true,
Base64Filename = "scrawl.png"
});
return(Json(new
{
fileid = result.FileId,
state = UpOp.GetStateMessage(result.State),
url = result.Url,
title = result.OriginFileName,
original = result.OriginFileName,
error = result.ErrorMessage
}, "text/html"));
}
case "uploadvideo":
{
UploadResult result = UpOp.Upload(new UploadConfig()
{
AllowExtensions = CustomConfig.VideoExts.ToArray(),
PathFormat = CustomConfig.EditorFilePath,
SizeLimit = Convert.ToInt32(CustomConfig.CanUploadFileSize * 1024 * 1024),
UploadFieldName = "upfile",
});
return(Json(new
{
fileid = result.FileId,
state = UpOp.GetStateMessage(result.State),
url = result.Url,
title = result.OriginFileName,
original = result.OriginFileName,
error = result.ErrorMessage
}, "text/html"));
}
case "uploadfile":
{
UploadResult result = UpOp.Upload(new UploadConfig()
{
AllowExtensions = CustomConfig.FileExts.ToArray(),
PathFormat = CustomConfig.EditorFilePath,
SizeLimit = Convert.ToInt32(CustomConfig.CanUploadFileSize * 1024 * 1024),
UploadFieldName = "upfile",
});
return(Json(new
{
fileid = result.FileId,
state = UpOp.GetStateMessage(result.State),
url = result.Url,
title = result.OriginFileName,
original = result.OriginFileName,
error = result.ErrorMessage
}, "text/html"));
}
case "listimage":
{
int Start = 0, Size = 20, Total = 300;
ResultState State = ResultState.Success;
IEnumerable <String> imgList = null;
Start = String.IsNullOrEmpty(Request["start"]) ? 1 : Convert.ToInt32(Request["start"]);
Size = String.IsNullOrEmpty(Request["size"]) ? 20 : Convert.ToInt32(Request["size"]);
try
{
var path = collection;
if (string.IsNullOrEmpty(collection))
{
path = CustomConfig.EditorImgPath + DateTime.Now.ToString("yyyy/MM/");
}
imgList = UpOp.GetFiles(path, "", Start, Size);
if (string.IsNullOrEmpty(collection))
{
imgList = imgList.Select(x => path + x);
}
}
catch (UnauthorizedAccessException)
{
State = ResultState.AuthorizError;
}
catch (DirectoryNotFoundException)
{
State = ResultState.PathNotFound;
}
catch (IOException)
{
State = ResultState.IOError;
}
return(Json(new
{
state = GetStateString(State),
list = imgList.Select(x => new { url = x }),
start = Start,
size = Size,
total = Total
}, JsonRequestBehavior.AllowGet));
}
case "listfile":
{
int Start = 0, Size = 20, Total = 300;
ResultState State = ResultState.Success;
IEnumerable <String> fileList = null;
Start = String.IsNullOrEmpty(Request["start"]) ? 0 : Convert.ToInt32(Request["start"]);
Size = String.IsNullOrEmpty(Request["size"]) ? 20 : Convert.ToInt32(Request["size"]);
try
{
var path = collection;
if (string.IsNullOrEmpty(collection))
{
path = CustomConfig.EditorImgPath + DateTime.Now.ToString("yyyy/MM/");
}
fileList = UpOp.GetFiles(path, "", Start, Size);
if (string.IsNullOrEmpty(collection))
{
fileList = fileList.Select(x => path + x);
}
}
catch (UnauthorizedAccessException)
{
State = ResultState.AuthorizError;
}
catch (DirectoryNotFoundException)
{
State = ResultState.PathNotFound;
}
catch (IOException)
{
State = ResultState.IOError;
}
return(Json(new
{
state = GetStateString(State),
list = fileList.Select(x => new { url = x }),
start = Start,
size = Size,
total = 0
}, JsonRequestBehavior.AllowGet));
}
case "catchimage":
{
string[] Sources;
CatchImg[] Crawlers;
Sources = Request.Form.GetValues("source[]");
if (Sources == null || Sources.Length == 0)
{
return(Json(new { state = "参数错误：没有指定抓取源" }));
}
else
{
Crawlers = Sources.Select(x => new CatchImg(x, context.Server).Fetch(CustomConfig.EditorImgPath)).ToArray();
}
return(Json(new
{
state = "SUCCESS",
list = Crawlers.Select(x => new
{
state = x.State,
source = x.SourceUrl,
url = x.ServerUrl
})
}, JsonRequestBehavior.AllowGet));
}
default:
return(Json(new { state = "action 参数为空或者 action 不被支持。" }));
}
}
