示例#1
0
        List <String> CreateRangeLinks(ArticleDate start, ArticleDate end)
        {
            pages  = new List <string>();
            years  = new List <string>();
            months = new List <string>();
            DateTime cursor = new DateTime(start.time.Ticks);

            downloaded      = new Dictionary <string, bool>();
            unwrapped       = new Dictionary <string, bool>();
            startedDownload = new Dictionary <string, bool>();
            synch           = new List <bool>();
            synch.Add(false);
            synch.Add(false);
            List <String> links = new List <String>();

            while (cursor.Ticks <= end.time.Ticks)
            {
                String line = bas;
                years.Add(cursor.Year.ToString());
                months.Add(cursor.Month.ToString());
                //if (cursor.Year < 2010) line += "0";
                line += cursor.Year.ToString() + "/" + cursor.Year.ToString() + "-";
                if (cursor.Month < 10)
                {
                    line += "0";
                }
                line += cursor.Month.ToString() + "/";
                pages.Add(line);
                cursor = PlusMonth(cursor);
            }

            /*update*/
            WebClient w = new WebClient();

            foreach (String pagee in pages)
            {
                String s = w.DownloadString(pagee);
                foreach (LinkItem li in LinkFinder.Find(s))
                {
                    if (li.Href[0] != 'p')
                    {
                        continue;
                    }
                    links.Add(pagee + li.Href);
                }
            }

            return(links);
        }
        public void DownloadStreamUpdate()
        {
            Console.WriteLine("Downloader started! My number is " + mynumber.ToString());

            // for (int i = 0; i < pages.Count; i++)
            foreach (String pagee in pages)
            {
                WebClient w = new WebClient();
                while (BunchHelper.outFile > 4)
                {
                    ;
                }
                while (BunchHelper.rawFile > 4)
                {
                    ;
                }
                String  s              = w.DownloadString(pagee);
                Int16   day            = 0;
                Boolean flag           = true;
                Int16   counter        = -1;
                char[]  delimiterChars = { '-', ' ' };
                Int32   index          = -1;

                foreach (LinkItem li in LinkFinder.Find(s))
                {
                    if (li.Href[0] != 'p')
                    {
                        continue;
                    }

                    if (day == 1 && flag)
                    {
                        flag = false;
                        continue;
                    }
                    ;
                    counter++;
                    String[] check          = li.Text.Split(delimiterChars);
                    Int16    validationHour = Int16.Parse(check[2].Substring(0, 2));
                    if (counter != validationHour)
                    {
                        counter = validationHour;
                    }
                    if (counter == 24)
                    {
                        counter = 0;
                    }
                    if (counter != 19)
                    {
                        continue;
                    }
                    String address = pagee + li.Href;
                    index++;
                    String to = @"d:\wp\code\3_wikiParserRealDigger\wikiParserRealDigger\bin\Debug\" + li.Href;// "_" + index.ToString();

                    day++;


                    if (BunchHelper.startedDownload.ContainsKey(li.Href))
                    {
                        continue;
                    }

                    BunchHelper.startedDownload.Add(li.Href, true);


                    using (var client = new WebClient())
                    {
                        BunchHelper.rawFile++;
                        // while (BunchHelper.rawFile >= 3)
                        // { ;}
                        Console.WriteLine(" I am downloading " + li.Href);
trymore:
                        try { client.DownloadFile(address, to); }
                        catch (System.Net.WebException ex) { System.Threading.Thread.Sleep(10000); goto trymore; }
                    }

                    BunchHelper.downloaded.Add(li.Href, true);
                    BunchHelper.starter = true;
                }
            }
            Console.WriteLine("Downloader ended");
            BunchHelper.downloadThreadEnd = true;
        }