示例#1
0
        protected void OnWebClientError(WebClientErrorEventArgs e)
        {
            EventHandler <WebClientErrorEventArgs> handler = WebClientError;

            if (handler != null)
            {
                handler(this, e);
            }
        }
示例#2
0
 protected void OnWebClientError(WebClientErrorEventArgs e)
 {
     EventHandler<WebClientErrorEventArgs> handler = WebClientError;
     if (handler != null)
     {
         handler(this, e);
     }
 }
示例#3
0
        /// <summary>
        /// 开始下载工作
        /// </summary>
        /// <param name="savePath">图片保存路径</param>
        public void StartDown(string savePath)
        {
            savePath.CheckNotNullOrEmpty("savePath");
            BaseUrl.CheckNotNullOrEmpty("BaseUrl" );
            WebClient.CheckNotNull("WebClient" );

            if (!Directory.Exists(savePath))
            {
                Directory.CreateDirectory(savePath);
            }

            InitializeForums(_forums);
            int total = 0;
            foreach (Forum tmpForum in _forums)
            {
                List<Group> groups = new List<Group>();
                string html;
                try
                {
                    html = WebClient.DownloadString(tmpForum.Url);
                }
                catch (Exception e)
                {
                    WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = tmpForum.Url, Message = e.Message };
                    OnWebClientError(arg);
                    continue;
                }
                Forum forum = PerfectForum(html, tmpForum);
                string forumPath = Path.Combine(savePath, forum.Name);
                if (!Directory.Exists(forumPath))
                {
                    Directory.CreateDirectory(forumPath);
                }
                foreach (Group @group in GetGroups(html, forum.Url))
                {
                    if (groups.All(m => m.Url != @group.Url))
                    {
                        groups.Add(@group);
                    }
                }
                if (forum.NextSearchMode == NextSearchMode.Cycle)
                {
                    forum.NextUrlFormat.CheckNotNullOrEmpty("forum.NextUrlFormat" );
                    for (int i = forum.SecondListNum; i <= forum.ListsCount - (2 - forum.SecondListNum); i++)
                    {
                        string forumUrl = string.Format(forum.NextUrlFormat, i);
                        try
                        {
                            html = WebClient.DownloadString(forumUrl);
                        }
                        catch (Exception e)
                        {
                            WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = forumUrl, Message = e.Message };
                            OnWebClientError(arg);
                            continue;
                        }
                        foreach (Group @group in GetGroups(html, forumUrl))
                        {
                            if (groups.All(m => m.Url != @group.Url))
                            {
                                groups.Add(@group);
                            }
                        }
                    }
                }
                else
                {
                    HasNextListPagePattern.CheckNotNullOrEmpty("HasNextListPagePattern" );
                    NextListPagePattern.CheckNotNullOrEmpty("NextListPagePattern" );
                    while (html.IsMatch(HasNextListPagePattern))
                    {
                        string forumUrl = html.Match(NextListPagePattern);
                        if (forumUrl.StartsWith("/"))
                        {
                            forumUrl = BaseUrl + forumUrl;
                        }
                        else if (!forumUrl.StartsWith("http"))
                        {
                            forumUrl = forum.Url.Substring(0, forum.Url.LastIndexOf("/", StringComparison.Ordinal) + 1) + forumUrl;
                        }
                        try
                        {
                            html = WebClient.DownloadString(forumUrl);
                        }
                        catch (Exception e)
                        {
                            WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = forumUrl, Message = e.Message };
                            OnWebClientError(arg);
                            break;
                        }
                        foreach (Group @group in GetGroups(html, forumUrl))
                        {
                            if (groups.All(m => m.Url != @group.Url))
                            {
                                groups.Add(@group);
                            }
                        }
                    }
                }
                forum.Groups.AddRange(groups);
                OnGroupGetCompleted(new GroupGetEventArgs { Forum = forum });
                int groupCount = 0;
                foreach (Group tmpGroup in groups)
                {
                    groupCount++;
                    try
                    {
                        html = WebClient.DownloadString(tmpGroup.Url);
                    }
                    catch (Exception e)
                    {
                        WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = tmpForum.Url, Message = e.Message };
                        OnWebClientError(arg);
                        continue;
                    }
                    Group group = PerfectGroup(html, tmpGroup);

                    OnGroupDownloading(new GroupDownloadEventArgs { Group = group });
                    List<string> images = GetImageUrls(html, group.Url).ToList();
                    foreach (string image in images)
                    {
                        if (!group.Images.Contains(image))
                        {
                            group.Images.Add(image);
                            OnImageUrlGetCompleted(new ImageUrlGetEventArgs { ForumName = forum.Name, GroupName = group.Name, ImageUrl = image });
                        }
                    }
                    if (group.NextSearchMode == NextSearchMode.Cycle)
                    {
                        group.NextUrlFormat.CheckNotNullOrEmpty("group.NextUrlFormat" );
                        for (int i = group.SecondImageNum; i < group.ImagesCount - (2 - group.SecondImageNum); i++)
                        {
                            string groupUrl = string.Format(group.NextUrlFormat, i);
                            try
                            {
                                html = WebClient.DownloadString(groupUrl);
                            }
                            catch (Exception e)
                            {
                                WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = groupUrl, Message = e.Message };
                                OnWebClientError(arg);
                                continue;
                            }
                            foreach (string image in GetImageUrls(html, group.Url))
                            {
                                if (!group.Images.Contains(image))
                                {
                                    group.Images.Add(image);
                                    OnImageUrlGetCompleted(new ImageUrlGetEventArgs { ForumName = forum.Name, GroupName = group.Name, ImageUrl = image });
                                }
                            }
                        }
                    }
                    else
                    {
                        HasNextImagePagePattern.CheckNotNullOrEmpty("HasNextImagePagePattern" );
                        NextImagePagePattern.CheckNotNullOrEmpty("NextImagePagePattern" );

                        while (html.IsMatch(HasNextImagePagePattern))
                        {
                            string groupUrl = html.Match(NextImagePagePattern);
                            if (groupUrl.StartsWith("/"))
                            {
                                groupUrl = BaseUrl + groupUrl;
                            }
                            else if (!groupUrl.StartsWith("http"))
                            {
                                groupUrl = group.Url.Substring(0, group.Url.LastIndexOf("/", StringComparison.Ordinal) + 1) + groupUrl;
                            }
                            try
                            {
                                html = WebClient.DownloadString(groupUrl);
                            }
                            catch (Exception e)
                            {
                                WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = groupUrl, Message = e.Message };
                                OnWebClientError(arg);
                                break;
                            }
                            foreach (string image in GetImageUrls(html, groupUrl))
                            {
                                if (!group.Images.Contains(image))
                                {
                                    group.Images.Add(image);
                                    OnImageUrlGetCompleted(new ImageUrlGetEventArgs { ForumName = forum.Name, GroupName = group.Name, ImageUrl = image });
                                }
                            }
                        }
                    }
                    int imageCount = 0;
                    foreach (string image in group.Images)
                    {
                        imageCount++;
                        string imageUrl = ImageDowning(image);
                        string ext = imageUrl.Substring(imageUrl.LastIndexOf('.')).ToLower();
                        string filename = string.Format("{0}.{1}-{2}{3}", groupCount, group.Name, imageCount.ToString("d3"),
                            (ext == ".jpg" || ext == ".png" || ext == ".bmp" || ext == ".gif") ? ext : ".jpg");
                        filename = Path.Combine(forumPath, filename);

                        try
                        {
                            ImageDownload(imageUrl, filename);
                            total++;
                            ImageDownloadEventArgs arg = new ImageDownloadEventArgs { Url = imageUrl, FileName = filename, Count = total };
                            OnImageDownloadCompleted(arg);
                        }
                        catch (Exception e)
                        {
                            WebClientErrorEventArgs arg = new WebClientErrorEventArgs { Url = imageUrl, Message = e.Message };
                            OnWebClientError(arg);
                        }
                    }
                    OnGroupDownloadCompleted(new GroupDownloadEventArgs { Group = group, Count = groupCount });
                }
                string forumFile = Path.Combine(savePath, forum.Name + ".xml");
                SerializeHelper.ToXmlFile(forum, forumFile);
                //if (!Directory.Exists(savePath))
                //{
                //    Directory.CreateDirectory(savePath);
                //}
                //TextWriter writer = File.CreateText(forumFile);
                //writer.WriteLine(forum.ToJsonString());
                //writer.Close();
            }
        }
示例#4
0
        /// <summary>
        /// 开始下载工作
        /// </summary>
        /// <param name="savePath">图片保存路径</param>
        public void StartDown(string savePath)
        {
            savePath.CheckNotNullOrEmpty("savePath");
            BaseUrl.CheckNotNullOrEmpty("BaseUrl");
            WebClient.CheckNotNull("WebClient");

            if (!Directory.Exists(savePath))
            {
                Directory.CreateDirectory(savePath);
            }

            InitializeForums(_forums);
            int total = 0;

            foreach (Forum tmpForum in _forums)
            {
                List <Group> groups = new List <Group>();
                string       html;
                try
                {
                    html = WebClient.DownloadString(tmpForum.Url);
                }
                catch (Exception e)
                {
                    WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
                        Url = tmpForum.Url, Message = e.Message
                    };
                    OnWebClientError(arg);
                    continue;
                }
                Forum  forum     = PerfectForum(html, tmpForum);
                string forumPath = Path.Combine(savePath, forum.Name);
                if (!Directory.Exists(forumPath))
                {
                    Directory.CreateDirectory(forumPath);
                }
                foreach (Group @group in GetGroups(html, forum.Url))
                {
                    if (groups.All(m => m.Url != @group.Url))
                    {
                        groups.Add(@group);
                    }
                }
                if (forum.NextSearchMode == NextSearchMode.Cycle)
                {
                    forum.NextUrlFormat.CheckNotNullOrEmpty("forum.NextUrlFormat");
                    for (int i = forum.SecondListNum; i <= forum.ListsCount - (2 - forum.SecondListNum); i++)
                    {
                        string forumUrl = string.Format(forum.NextUrlFormat, i);
                        try
                        {
                            html = WebClient.DownloadString(forumUrl);
                        }
                        catch (Exception e)
                        {
                            WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
                                Url = forumUrl, Message = e.Message
                            };
                            OnWebClientError(arg);
                            continue;
                        }
                        foreach (Group @group in GetGroups(html, forumUrl))
                        {
                            if (groups.All(m => m.Url != @group.Url))
                            {
                                groups.Add(@group);
                            }
                        }
                    }
                }
                else
                {
                    HasNextListPagePattern.CheckNotNullOrEmpty("HasNextListPagePattern");
                    NextListPagePattern.CheckNotNullOrEmpty("NextListPagePattern");
                    while (html.IsMatch(HasNextListPagePattern))
                    {
                        string forumUrl = html.Match(NextListPagePattern);
                        if (forumUrl.StartsWith("/"))
                        {
                            forumUrl = BaseUrl + forumUrl;
                        }
                        else if (!forumUrl.StartsWith("http"))
                        {
                            forumUrl = forum.Url.Substring(0, forum.Url.LastIndexOf("/", StringComparison.Ordinal) + 1) + forumUrl;
                        }
                        try
                        {
                            html = WebClient.DownloadString(forumUrl);
                        }
                        catch (Exception e)
                        {
                            WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
                                Url = forumUrl, Message = e.Message
                            };
                            OnWebClientError(arg);
                            break;
                        }
                        foreach (Group @group in GetGroups(html, forumUrl))
                        {
                            if (groups.All(m => m.Url != @group.Url))
                            {
                                groups.Add(@group);
                            }
                        }
                    }
                }
                forum.Groups.AddRange(groups);
                OnGroupGetCompleted(new GroupGetEventArgs {
                    Forum = forum
                });
                int groupCount = 0;
                foreach (Group tmpGroup in groups)
                {
                    groupCount++;
                    try
                    {
                        html = WebClient.DownloadString(tmpGroup.Url);
                    }
                    catch (Exception e)
                    {
                        WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
                            Url = tmpForum.Url, Message = e.Message
                        };
                        OnWebClientError(arg);
                        continue;
                    }
                    Group group = PerfectGroup(html, tmpGroup);

                    OnGroupDownloading(new GroupDownloadEventArgs {
                        Group = group
                    });
                    List <string> images = GetImageUrls(html, group.Url).ToList();
                    foreach (string image in images)
                    {
                        if (!group.Images.Contains(image))
                        {
                            group.Images.Add(image);
                            OnImageUrlGetCompleted(new ImageUrlGetEventArgs {
                                ForumName = forum.Name, GroupName = group.Name, ImageUrl = image
                            });
                        }
                    }
                    if (group.NextSearchMode == NextSearchMode.Cycle)
                    {
                        group.NextUrlFormat.CheckNotNullOrEmpty("group.NextUrlFormat");
                        for (int i = group.SecondImageNum; i < group.ImagesCount - (2 - group.SecondImageNum); i++)
                        {
                            string groupUrl = string.Format(group.NextUrlFormat, i);
                            try
                            {
                                html = WebClient.DownloadString(groupUrl);
                            }
                            catch (Exception e)
                            {
                                WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
                                    Url = groupUrl, Message = e.Message
                                };
                                OnWebClientError(arg);
                                continue;
                            }
                            foreach (string image in GetImageUrls(html, group.Url))
                            {
                                if (!group.Images.Contains(image))
                                {
                                    group.Images.Add(image);
                                    OnImageUrlGetCompleted(new ImageUrlGetEventArgs {
                                        ForumName = forum.Name, GroupName = group.Name, ImageUrl = image
                                    });
                                }
                            }
                        }
                    }
                    else
                    {
                        HasNextImagePagePattern.CheckNotNullOrEmpty("HasNextImagePagePattern");
                        NextImagePagePattern.CheckNotNullOrEmpty("NextImagePagePattern");

                        while (html.IsMatch(HasNextImagePagePattern))
                        {
                            string groupUrl = html.Match(NextImagePagePattern);
                            if (groupUrl.StartsWith("/"))
                            {
                                groupUrl = BaseUrl + groupUrl;
                            }
                            else if (!groupUrl.StartsWith("http"))
                            {
                                groupUrl = group.Url.Substring(0, group.Url.LastIndexOf("/", StringComparison.Ordinal) + 1) + groupUrl;
                            }
                            try
                            {
                                html = WebClient.DownloadString(groupUrl);
                            }
                            catch (Exception e)
                            {
                                WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
                                    Url = groupUrl, Message = e.Message
                                };
                                OnWebClientError(arg);
                                break;
                            }
                            foreach (string image in GetImageUrls(html, groupUrl))
                            {
                                if (!group.Images.Contains(image))
                                {
                                    group.Images.Add(image);
                                    OnImageUrlGetCompleted(new ImageUrlGetEventArgs {
                                        ForumName = forum.Name, GroupName = group.Name, ImageUrl = image
                                    });
                                }
                            }
                        }
                    }
                    int imageCount = 0;
                    foreach (string image in group.Images)
                    {
                        imageCount++;
                        string imageUrl = ImageDowning(image);
                        string ext      = imageUrl.Substring(imageUrl.LastIndexOf('.')).ToLower();
                        string filename = string.Format("{0}.{1}-{2}{3}", groupCount, group.Name, imageCount.ToString("d3"),
                                                        (ext == ".jpg" || ext == ".png" || ext == ".bmp" || ext == ".gif") ? ext : ".jpg");
                        filename = Path.Combine(forumPath, filename);

                        try
                        {
                            ImageDownload(imageUrl, filename);
                            total++;
                            ImageDownloadEventArgs arg = new ImageDownloadEventArgs {
                                Url = imageUrl, FileName = filename, Count = total
                            };
                            OnImageDownloadCompleted(arg);
                        }
                        catch (Exception e)
                        {
                            WebClientErrorEventArgs arg = new WebClientErrorEventArgs {
                                Url = imageUrl, Message = e.Message
                            };
                            OnWebClientError(arg);
                        }
                    }
                    OnGroupDownloadCompleted(new GroupDownloadEventArgs {
                        Group = group, Count = groupCount
                    });
                }
                string forumFile = Path.Combine(savePath, forum.Name + ".xml");
                SerializeHelper.ToXmlFile(forum, forumFile);
                //if (!Directory.Exists(savePath))
                //{
                //    Directory.CreateDirectory(savePath);
                //}
                //TextWriter writer = File.CreateText(forumFile);
                //writer.WriteLine(forum.ToJsonString());
                //writer.Close();
            }
        }