Ejemplo n.º 1
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="pURL">目标URL,必须参数</param>
        /// <param name="pFileName">存名称,默认为空,表示由迅雷处理,可选参数</param>
        /// <param name="pPath">存储目录,默认为空,表示由迅雷处理,可选参数</param>
        /// <param name="pComments">下载注释,默认为空,可选参数</param>
        /// <param name="pReferURL">引用页URL,默认为空,可选参数</param>
        /// <param name="nStartMode">开始模式,0手工开始,1立即开始,默认为 - 1,表示由迅雷处理,可选参数</param>
        /// <param name="nOnlyFromOrigin">是否只从原始URL下载,1只从原始URL下载,0多资源下载,默认为0,可选参数</param>
        /// <param name="nOriginThreadCount">原始地址下载线程数,范围1 - 10,默认为 - 1,表示由迅雷处理,可选参数</param>
        static void AddTask(string pURL, string pFileName, string pPath, string pComments, string pReferURL, int nStartMode, int nOnlyFromOrigin, int nOriginThreadCount)
        {
            ThunderAgentLib.AgentClass agentClass = new ThunderAgentLib.AgentClass();

            //添加任务:下载http://www.baidu.com/index.html这个文件至C:\baidu.html,
            //没有注释,没有引用,立即开始,从多资源下载,原始资源线程5
            agentClass.AddTask(pURL, pFileName, pPath, pComments, pReferURL, nStartMode, nOnlyFromOrigin, nOriginThreadCount);
            agentClass.CommitTasks2(1);//提交
        }
Ejemplo n.º 2
0
        static void Main(string[] args)
        {
            //2014年3月18日获取至4110
            //2014年4月18日获取至4174
            //2014年5月18日获取至4223
            //2014年6月23日获取至4296
            //2014年7月30日获取至4380
            //2014年8月21日获取至4433
            #region Step 1: 找出所有存在的页面(即返回代码为200的),把生成的程序放在多个文件夹下同时跑,程序运行结束后在文件夹下会得到url.txt,里面保存着所有存在的页面链接;2014年3月18日最新页面为4110
            //StreamWriter fw = new StreamWriter("url.txt");
            //string baseUrl = "http://www.meizitu.com/a/";
            //for (int i = 4381; i < 4434; i++)
            //{
            //    string url = baseUrl + i + ".html";
            //    try
            //    {
            //        HttpWebRequest httpWebRequest = HttpWebRequest.CreateHttp(url);
            //        httpWebRequest.Method = "GET";
            //        HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
            //        if (httpWebResponse.StatusCode.Equals(HttpStatusCode.OK))
            //        {
            //            fw.WriteLine(url);
            //        }
            //        httpWebResponse.Close();
            //    }
            //    catch (WebException ex)
            //    {
            //        HttpWebResponse response = (HttpWebResponse)ex.Response;
            //        if (response != null)  //排除对象为空的错误
            //        {
            //            Console.WriteLine(response.StatusCode);
            //            response.Close();
            //        }
            //    }
            //    finally
            //    {
            //        Console.WriteLine(i);
            //    }
            //}
            //fw.Close();
            #endregion

            #region Step 2: 根据url.txt爬取页面中的妹子图片
            ThunderAgentLib.AgentClass agent = new ThunderAgentLib.AgentClass();
            StreamReader  fr    = new StreamReader("url.txt");
            List <string> links = new List <string>();
            while (!fr.EndOfStream)
            {
                links.Add(fr.ReadLine());
            }
            fr.Close();
            foreach (string link in links)
            {
                try
                {
                    HttpWebRequest httpWebRequest = HttpWebRequest.CreateHttp(link);
                    httpWebRequest.Method = "GET";
                    HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
                    StreamReader    reader          = new StreamReader(new BufferedStream(httpWebResponse.GetResponseStream(), 4 * 200 * 1024), Encoding.GetEncoding("gb2312"));
                    string          htmlContent     = reader.ReadToEnd();
                    httpWebResponse.Close();
                    reader.Close();
                    int startIndex = 0;
                    startIndex = htmlContent.IndexOf("<title>");
                    int           endIndex = htmlContent.IndexOf(" | 妹子图");
                    string        title    = htmlContent.Substring(startIndex + 7, endIndex - startIndex - 7);
                    List <string> picLinks = new List <string>();
                    do
                    {
                        startIndex = htmlContent.IndexOf("src=\"http://www.meizitu.com/wp-content/uploads/", startIndex);
                        if (startIndex != -1)
                        {
                            endIndex    = htmlContent.IndexOf(".jpg", startIndex);
                            startIndex += 5;
                            string picLink = htmlContent.Substring(startIndex, endIndex + 4 - startIndex);
                            if (picLink.IndexOf("limg.jpg") == -1 && picLink.IndexOf("hezuo") == -1)
                            {
                                picLinks.Add(picLink);
                            }
                        }
                        else
                        {
                            break;
                        }
                    } while (true);
                    int picLinkIndex = 0;
                    foreach (string picLink in picLinks)
                    {
                        //string fileName = basePath + title + "_" + picLinkIndex + ".jpg";
                        string fileName = title + "_" + picLinkIndex + ".jpg";
                        if (!shouldDownloadSet.ContainsKey(picLink))
                        {
                            agent.AddTask(picLink, fileName, "D:\\Download\\", "", "", 1, 0, 1);
                            shouldDownloadSet.Add(picLink, fileName);
                            //WebClient wc = new WebClient();
                            //wc.DownloadFileAsync(new Uri(picLink), fileName);
                            //wc.DownloadFileCompleted += wc_DownloadFileCompleted;
                        }
                        picLinkIndex++;
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine(link + "出错!");
                }
            }
            Console.WriteLine("共找到" + shouldDownloadSet.Count + "张图片!");
            agent.CommitTasks2(1);
            Console.WriteLine("开始使用迅雷下载图片!等待下载完成……");
            Console.Read();

            //此部分代码作用是把要下载的图片链接和文件名的映射保存到文件Download.txt中,手工拷入迅雷中下载;在调用迅雷的API后可能没有必要了
            //StreamWriter fw = new StreamWriter("Download.txt");
            //foreach (KeyValuePair<string, string> fileName in shouldDownloadSet)
            //{
            //    fw.WriteLine(fileName.Key + " " + fileName.Value);
            //}
            //fw.Close();
            //StreamWriter fw2 = new StreamWriter("Download2.txt");
            //foreach (KeyValuePair<string, string> fileName in shouldDownloadSet)
            //{
            //    fw2.WriteLine(fileName.Key);
            //}
            //fw2.Close();
            #endregion

            #region Step 3: 根据Download.txt和保存图片的文件夹,重新下载文件大小为0的图片
            //string[] fileNameList = Directory.GetFiles("MeiZiTu/");
            //List<string> needReDownloadFileList = new List<string>();
            //foreach (string fileName in fileNameList)
            //{
            //    FileInfo file = new FileInfo(fileName);
            //    if (file.Length == 0)
            //    {
            //        needReDownloadFileList.Add(file.Name);
            //    }
            //}
            //StreamReader fr = new StreamReader("Download.txt");
            //while (!fr.EndOfStream)
            //{
            //    string rawStr = fr.ReadLine();
            //    int splitIndex = rawStr.IndexOf(".jpg MeiZiTu");
            //    string key = rawStr.Substring(0, splitIndex + 4);
            //    string value = rawStr.Substring(splitIndex + 13);
            //    if (!shouldDownloadSet.ContainsKey(value))
            //    {
            //        shouldDownloadSet.Add(value, key);
            //    }
            //}
            //foreach (string fileName in needReDownloadFileList)
            //{
            //    string url = shouldDownloadSet[fileName];
            //    WebClient wc = new WebClient();
            //    wc.DownloadFileAsync(new Uri(url), fileName);
            //    wc.DownloadFileCompleted += wc_DownloadFileCompleted;
            //}
            //Console.Read();
            #endregion
        }
Ejemplo n.º 3
0
        static void Main(string[] args)
        {
            //2014年3月18日获取至4110
            //2014年4月18日获取至4174
            //2014年5月18日获取至4223
            //2014年6月23日获取至4296
            //2014年7月30日获取至4380
            //2014年8月21日获取至4433
            #region Step 1: 找出所有存在的页面(即返回代码为200的),把生成的程序放在多个文件夹下同时跑,程序运行结束后在文件夹下会得到url.txt,里面保存着所有存在的页面链接;2014年3月18日最新页面为4110
            //StreamWriter fw = new StreamWriter("url.txt");
            //string baseUrl = "http://www.meizitu.com/a/";
            //for (int i = 4381; i < 4434; i++)
            //{
            //    string url = baseUrl + i + ".html";
            //    try
            //    {
            //        HttpWebRequest httpWebRequest = HttpWebRequest.CreateHttp(url);
            //        httpWebRequest.Method = "GET";
            //        HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
            //        if (httpWebResponse.StatusCode.Equals(HttpStatusCode.OK))
            //        {
            //            fw.WriteLine(url);
            //        }
            //        httpWebResponse.Close();
            //    }
            //    catch (WebException ex)
            //    {
            //        HttpWebResponse response = (HttpWebResponse)ex.Response;
            //        if (response != null)  //排除对象为空的错误
            //        {
            //            Console.WriteLine(response.StatusCode);
            //            response.Close();
            //        }
            //    }
            //    finally
            //    {
            //        Console.WriteLine(i);
            //    }
            //}
            //fw.Close();
            #endregion

            #region Step 2: 根据url.txt爬取页面中的妹子图片
            ThunderAgentLib.AgentClass agent = new ThunderAgentLib.AgentClass();
            StreamReader fr = new StreamReader("url.txt");
            List<string> links = new List<string>();
            while (!fr.EndOfStream)
            {
                links.Add(fr.ReadLine());
            }
            fr.Close();
            foreach (string link in links)
            {
                try
                {
                    HttpWebRequest httpWebRequest = HttpWebRequest.CreateHttp(link);
                    httpWebRequest.Method = "GET";
                    HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
                    StreamReader reader = new StreamReader(new BufferedStream(httpWebResponse.GetResponseStream(), 4 * 200 * 1024), Encoding.GetEncoding("gb2312"));
                    string htmlContent = reader.ReadToEnd();
                    httpWebResponse.Close();
                    reader.Close();
                    int startIndex = 0;
                    startIndex = htmlContent.IndexOf("<title>");
                    int endIndex = htmlContent.IndexOf(" | 妹子图");
                    string title = htmlContent.Substring(startIndex + 7, endIndex - startIndex - 7);
                    List<string> picLinks = new List<string>();
                    do
                    {
                        startIndex = htmlContent.IndexOf("src=\"http://www.meizitu.com/wp-content/uploads/", startIndex);
                        if (startIndex != -1)
                        {
                            endIndex = htmlContent.IndexOf(".jpg", startIndex);
                            startIndex += 5;
                            string picLink = htmlContent.Substring(startIndex, endIndex + 4 - startIndex);
                            if (picLink.IndexOf("limg.jpg") == -1 && picLink.IndexOf("hezuo") == -1)
                            {
                                picLinks.Add(picLink);
                            }
                        }
                        else
                        {
                            break;
                        }
                    } while (true);
                    int picLinkIndex = 0;
                    foreach (string picLink in picLinks)
                    {
                        //string fileName = basePath + title + "_" + picLinkIndex + ".jpg";
                        string fileName = title + "_" + picLinkIndex + ".jpg";
                        if (!shouldDownloadSet.ContainsKey(picLink))
                        {
                            agent.AddTask(picLink, fileName, "D:\\Download\\", "", "", 1, 0, 1);
                            shouldDownloadSet.Add(picLink, fileName);
                            //WebClient wc = new WebClient();
                            //wc.DownloadFileAsync(new Uri(picLink), fileName);
                            //wc.DownloadFileCompleted += wc_DownloadFileCompleted;
                        }
                        picLinkIndex++;
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine(link + "出错!");
                }
            }
            Console.WriteLine("共找到" + shouldDownloadSet.Count + "张图片!");
            agent.CommitTasks2(1);
            Console.WriteLine("开始使用迅雷下载图片!等待下载完成……");
            Console.Read();

            //此部分代码作用是把要下载的图片链接和文件名的映射保存到文件Download.txt中,手工拷入迅雷中下载;在调用迅雷的API后可能没有必要了
            //StreamWriter fw = new StreamWriter("Download.txt");
            //foreach (KeyValuePair<string, string> fileName in shouldDownloadSet)
            //{
            //    fw.WriteLine(fileName.Key + " " + fileName.Value);
            //}
            //fw.Close();
            //StreamWriter fw2 = new StreamWriter("Download2.txt");
            //foreach (KeyValuePair<string, string> fileName in shouldDownloadSet)
            //{
            //    fw2.WriteLine(fileName.Key);
            //}
            //fw2.Close();
            #endregion

            #region Step 3: 根据Download.txt和保存图片的文件夹,重新下载文件大小为0的图片
            //string[] fileNameList = Directory.GetFiles("MeiZiTu/");
            //List<string> needReDownloadFileList = new List<string>();
            //foreach (string fileName in fileNameList)
            //{
            //    FileInfo file = new FileInfo(fileName);
            //    if (file.Length == 0)
            //    {
            //        needReDownloadFileList.Add(file.Name);
            //    }
            //}
            //StreamReader fr = new StreamReader("Download.txt");
            //while (!fr.EndOfStream)
            //{
            //    string rawStr = fr.ReadLine();
            //    int splitIndex = rawStr.IndexOf(".jpg MeiZiTu");
            //    string key = rawStr.Substring(0, splitIndex + 4);
            //    string value = rawStr.Substring(splitIndex + 13);
            //    if (!shouldDownloadSet.ContainsKey(value))
            //    {
            //        shouldDownloadSet.Add(value, key);
            //    }
            //}
            //foreach (string fileName in needReDownloadFileList)
            //{
            //    string url = shouldDownloadSet[fileName];
            //    WebClient wc = new WebClient();
            //    wc.DownloadFileAsync(new Uri(url), fileName);
            //    wc.DownloadFileCompleted += wc_DownloadFileCompleted;
            //}
            //Console.Read();
            #endregion
        }