protected void OnWebClientError(WebClientErrorEventArgs e)
{
EventHandler <WebClientErrorEventArgs> handler = WebClientError;
if (handler != null)
{
handler(this, e);
}
}
protected void OnWebClientError(WebClientErrorEventArgs e)
{
EventHandler<WebClientErrorEventArgs> handler = WebClientError;
if (handler != null)
{
handler(this, e);
}
}
/// <summary>
/// 开始下载工作
/// </summary>
/// <param name="savePath">图片保存路径</param>
public void StartDown(string savePath)
{
savePath.CheckNotNullOrEmpty("savePath");
BaseUrl.CheckNotNullOrEmpty("BaseUrl" );
WebClient.CheckNotNull("WebClient" );
if (!Directory.Exists(savePath))
{
Directory.CreateDirectory(savePath);
}
InitializeForums(_forums);
int total = 0;
foreach (Forum tmpForum in _forums)
{
List<Group> groups = new List<Group>();
string html;
try
{
html = WebClient.DownloadString(tmpForum.Url);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = tmpForum.Url, Message = e.Message };
OnWebClientError(arg);
continue;
}
Forum forum = PerfectForum(html, tmpForum);
string forumPath = Path.Combine(savePath, forum.Name);
if (!Directory.Exists(forumPath))
{
Directory.CreateDirectory(forumPath);
}
foreach (Group @group in GetGroups(html, forum.Url))
{
if (groups.All(m => m.Url != @group.Url))
{
groups.Add(@group);
}
}
if (forum.NextSearchMode == NextSearchMode.Cycle)
{
forum.NextUrlFormat.CheckNotNullOrEmpty("forum.NextUrlFormat" );
for (int i = forum.SecondListNum; i <= forum.ListsCount - (2 - forum.SecondListNum); i++)
{
string forumUrl = string.Format(forum.NextUrlFormat, i);
try
{
html = WebClient.DownloadString(forumUrl);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = forumUrl, Message = e.Message };
OnWebClientError(arg);
continue;
}
foreach (Group @group in GetGroups(html, forumUrl))
{
if (groups.All(m => m.Url != @group.Url))
{
groups.Add(@group);
}
}
}
}
else
{
HasNextListPagePattern.CheckNotNullOrEmpty("HasNextListPagePattern" );
NextListPagePattern.CheckNotNullOrEmpty("NextListPagePattern" );
while (html.IsMatch(HasNextListPagePattern))
{
string forumUrl = html.Match(NextListPagePattern);
if (forumUrl.StartsWith("/"))
{
forumUrl = BaseUrl + forumUrl;
}
else if (!forumUrl.StartsWith("http"))
{
forumUrl = forum.Url.Substring(0, forum.Url.LastIndexOf("/", StringComparison.Ordinal) + 1) + forumUrl;
}
try
{
html = WebClient.DownloadString(forumUrl);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = forumUrl, Message = e.Message };
OnWebClientError(arg);
break;
}
foreach (Group @group in GetGroups(html, forumUrl))
{
if (groups.All(m => m.Url != @group.Url))
{
groups.Add(@group);
}
}
}
}
forum.Groups.AddRange(groups);
OnGroupGetCompleted(new GroupGetEventArgs { Forum = forum });
int groupCount = 0;
foreach (Group tmpGroup in groups)
{
groupCount++;
try
{
html = WebClient.DownloadString(tmpGroup.Url);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = tmpForum.Url, Message = e.Message };
OnWebClientError(arg);
continue;
}
Group group = PerfectGroup(html, tmpGroup);
OnGroupDownloading(new GroupDownloadEventArgs { Group = group });
List<string> images = GetImageUrls(html, group.Url).ToList();
foreach (string image in images)
{
if (!group.Images.Contains(image))
{
group.Images.Add(image);
OnImageUrlGetCompleted(new ImageUrlGetEventArgs { ForumName = forum.Name, GroupName = group.Name, ImageUrl = image });
}
}
if (group.NextSearchMode == NextSearchMode.Cycle)
{
group.NextUrlFormat.CheckNotNullOrEmpty("group.NextUrlFormat" );
for (int i = group.SecondImageNum; i < group.ImagesCount - (2 - group.SecondImageNum); i++)
{
string groupUrl = string.Format(group.NextUrlFormat, i);
try
{
html = WebClient.DownloadString(groupUrl);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = groupUrl, Message = e.Message };
OnWebClientError(arg);
continue;
}
foreach (string image in GetImageUrls(html, group.Url))
{
if (!group.Images.Contains(image))
{
group.Images.Add(image);
OnImageUrlGetCompleted(new ImageUrlGetEventArgs { ForumName = forum.Name, GroupName = group.Name, ImageUrl = image });
}
}
}
}
else
{
HasNextImagePagePattern.CheckNotNullOrEmpty("HasNextImagePagePattern" );
NextImagePagePattern.CheckNotNullOrEmpty("NextImagePagePattern" );
while (html.IsMatch(HasNextImagePagePattern))
{
string groupUrl = html.Match(NextImagePagePattern);
if (groupUrl.StartsWith("/"))
{
groupUrl = BaseUrl + groupUrl;
}
else if (!groupUrl.StartsWith("http"))
{
groupUrl = group.Url.Substring(0, group.Url.LastIndexOf("/", StringComparison.Ordinal) + 1) + groupUrl;
}
try
{
html = WebClient.DownloadString(groupUrl);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = groupUrl, Message = e.Message };
OnWebClientError(arg);
break;
}
foreach (string image in GetImageUrls(html, groupUrl))
{
if (!group.Images.Contains(image))
{
group.Images.Add(image);
OnImageUrlGetCompleted(new ImageUrlGetEventArgs { ForumName = forum.Name, GroupName = group.Name, ImageUrl = image });
}
}
}
}
int imageCount = 0;
foreach (string image in group.Images)
{
imageCount++;
string imageUrl = ImageDowning(image);
string ext = imageUrl.Substring(imageUrl.LastIndexOf('.')).ToLower();
string filename = string.Format("{0}.{1}-{2}{3}", groupCount, group.Name, imageCount.ToString("d3"),
(ext == ".jpg" || ext == ".png" || ext == ".bmp" || ext == ".gif") ? ext : ".jpg");
filename = Path.Combine(forumPath, filename);
try
{
ImageDownload(imageUrl, filename);
total++;
ImageDownloadEventArgs arg = new ImageDownloadEventArgs { Url = imageUrl, FileName = filename, Count = total };
OnImageDownloadCompleted(arg);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = imageUrl, Message = e.Message };
OnWebClientError(arg);
}
}
OnGroupDownloadCompleted(new GroupDownloadEventArgs { Group = group, Count = groupCount });
}
string forumFile = Path.Combine(savePath, forum.Name + ".xml");
SerializeHelper.ToXmlFile(forum, forumFile);
//if (!Directory.Exists(savePath))
//{
// Directory.CreateDirectory(savePath);
//}
//TextWriter writer = File.CreateText(forumFile);
//writer.WriteLine(forum.ToJsonString());
//writer.Close();
}
}
/// <summary>
/// 开始下载工作
/// </summary>
/// <param name="savePath">图片保存路径</param>
public void StartDown(string savePath)
{
savePath.CheckNotNullOrEmpty("savePath");
BaseUrl.CheckNotNullOrEmpty("BaseUrl");
WebClient.CheckNotNull("WebClient");
if (!Directory.Exists(savePath))
{
Directory.CreateDirectory(savePath);
}
InitializeForums(_forums);
int total = 0;
foreach (Forum tmpForum in _forums)
{
List <Group> groups = new List <Group>();
string html;
try
{
html = WebClient.DownloadString(tmpForum.Url);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
Url = tmpForum.Url, Message = e.Message
};
OnWebClientError(arg);
continue;
}
Forum forum = PerfectForum(html, tmpForum);
string forumPath = Path.Combine(savePath, forum.Name);
if (!Directory.Exists(forumPath))
{
Directory.CreateDirectory(forumPath);
}
foreach (Group @group in GetGroups(html, forum.Url))
{
if (groups.All(m => m.Url != @group.Url))
{
groups.Add(@group);
}
}
if (forum.NextSearchMode == NextSearchMode.Cycle)
{
forum.NextUrlFormat.CheckNotNullOrEmpty("forum.NextUrlFormat");
for (int i = forum.SecondListNum; i <= forum.ListsCount - (2 - forum.SecondListNum); i++)
{
string forumUrl = string.Format(forum.NextUrlFormat, i);
try
{
html = WebClient.DownloadString(forumUrl);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
Url = forumUrl, Message = e.Message
};
OnWebClientError(arg);
continue;
}
foreach (Group @group in GetGroups(html, forumUrl))
{
if (groups.All(m => m.Url != @group.Url))
{
groups.Add(@group);
}
}
}
}
else
{
HasNextListPagePattern.CheckNotNullOrEmpty("HasNextListPagePattern");
NextListPagePattern.CheckNotNullOrEmpty("NextListPagePattern");
while (html.IsMatch(HasNextListPagePattern))
{
string forumUrl = html.Match(NextListPagePattern);
if (forumUrl.StartsWith("/"))
{
forumUrl = BaseUrl + forumUrl;
}
else if (!forumUrl.StartsWith("http"))
{
forumUrl = forum.Url.Substring(0, forum.Url.LastIndexOf("/", StringComparison.Ordinal) + 1) + forumUrl;
}
try
{
html = WebClient.DownloadString(forumUrl);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
Url = forumUrl, Message = e.Message
};
OnWebClientError(arg);
break;
}
foreach (Group @group in GetGroups(html, forumUrl))
{
if (groups.All(m => m.Url != @group.Url))
{
groups.Add(@group);
}
}
}
}
forum.Groups.AddRange(groups);
OnGroupGetCompleted(new GroupGetEventArgs {
Forum = forum
});
int groupCount = 0;
foreach (Group tmpGroup in groups)
{
groupCount++;
try
{
html = WebClient.DownloadString(tmpGroup.Url);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
Url = tmpForum.Url, Message = e.Message
};
OnWebClientError(arg);
continue;
}
Group group = PerfectGroup(html, tmpGroup);
OnGroupDownloading(new GroupDownloadEventArgs {
Group = group
});
List <string> images = GetImageUrls(html, group.Url).ToList();
foreach (string image in images)
{
if (!group.Images.Contains(image))
{
group.Images.Add(image);
OnImageUrlGetCompleted(new ImageUrlGetEventArgs {
ForumName = forum.Name, GroupName = group.Name, ImageUrl = image
});
}
}
if (group.NextSearchMode == NextSearchMode.Cycle)
{
group.NextUrlFormat.CheckNotNullOrEmpty("group.NextUrlFormat");
for (int i = group.SecondImageNum; i < group.ImagesCount - (2 - group.SecondImageNum); i++)
{
string groupUrl = string.Format(group.NextUrlFormat, i);
try
{
html = WebClient.DownloadString(groupUrl);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
Url = groupUrl, Message = e.Message
};
OnWebClientError(arg);
continue;
}
foreach (string image in GetImageUrls(html, group.Url))
{
if (!group.Images.Contains(image))
{
group.Images.Add(image);
OnImageUrlGetCompleted(new ImageUrlGetEventArgs {
ForumName = forum.Name, GroupName = group.Name, ImageUrl = image
});
}
}
}
}
else
{
HasNextImagePagePattern.CheckNotNullOrEmpty("HasNextImagePagePattern");
NextImagePagePattern.CheckNotNullOrEmpty("NextImagePagePattern");
while (html.IsMatch(HasNextImagePagePattern))
{
string groupUrl = html.Match(NextImagePagePattern);
if (groupUrl.StartsWith("/"))
{
groupUrl = BaseUrl + groupUrl;
}
else if (!groupUrl.StartsWith("http"))
{
groupUrl = group.Url.Substring(0, group.Url.LastIndexOf("/", StringComparison.Ordinal) + 1) + groupUrl;
}
try
{
html = WebClient.DownloadString(groupUrl);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
Url = groupUrl, Message = e.Message
};
OnWebClientError(arg);
break;
}
foreach (string image in GetImageUrls(html, groupUrl))
{
if (!group.Images.Contains(image))
{
group.Images.Add(image);
OnImageUrlGetCompleted(new ImageUrlGetEventArgs {
ForumName = forum.Name, GroupName = group.Name, ImageUrl = image
});
}
}
}
}
int imageCount = 0;
foreach (string image in group.Images)
{
imageCount++;
string imageUrl = ImageDowning(image);
string ext = imageUrl.Substring(imageUrl.LastIndexOf('.')).ToLower();
string filename = string.Format("{0}.{1}-{2}{3}", groupCount, group.Name, imageCount.ToString("d3"),
(ext == ".jpg" || ext == ".png" || ext == ".bmp" || ext == ".gif") ? ext : ".jpg");
filename = Path.Combine(forumPath, filename);
try
{
ImageDownload(imageUrl, filename);
total++;
ImageDownloadEventArgs arg = new ImageDownloadEventArgs {
Url = imageUrl, FileName = filename, Count = total
};
OnImageDownloadCompleted(arg);
}
catch (Exception e)
{
WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
Url = imageUrl, Message = e.Message
};
OnWebClientError(arg);
}
}
OnGroupDownloadCompleted(new GroupDownloadEventArgs {
Group = group, Count = groupCount
});
}
string forumFile = Path.Combine(savePath, forum.Name + ".xml");
SerializeHelper.ToXmlFile(forum, forumFile);
//if (!Directory.Exists(savePath))
//{
// Directory.CreateDirectory(savePath);
//}
//TextWriter writer = File.CreateText(forumFile);
//writer.WriteLine(forum.ToJsonString());
//writer.Close();
}
}
