void CreateRange(ArticleDate start, ArticleDate end) { pages = new List <string>(); years = new List <string>(); months = new List <string>(); DateTime cursor = new DateTime(start.time.Ticks); downloaded = new Dictionary <string, bool>(); unwrapped = new Dictionary <string, bool>(); startedDownload = new Dictionary <string, bool>(); synch = new List <bool>(); synch.Add(false); synch.Add(false); while (cursor.Ticks <= end.time.Ticks) { String line = bas; years.Add(cursor.Year.ToString()); months.Add(cursor.Month.ToString()); //if (cursor.Year < 2010) line += "0"; line += cursor.Year.ToString() + "/" + cursor.Year.ToString() + "-"; if (cursor.Month < 10) { line += "0"; } line += cursor.Month.ToString() + "/"; pages.Add(line); cursor = PlusMonth(cursor); } }
public void processPipeFile(String[] file, String dd = "") { try { RulesDictionary.CreateSelectedList(@"D:\wp\pipe\treatmentinput\treatmentgroups.txt"); } catch (Exception ex) { } start_ = DateTime.ParseExact(Start, "MM/yyyy", null); end_ = DateTime.ParseExact(End, "MM/yyyy", null); start = new ArticleDate(DateTime.ParseExact(Start, "MM/yyyy", null)); end = new ArticleDate(DateTime.ParseExact(End, "MM/yyyy", null)); borns = new Dictionary <Int64, DateTime>(); pagesizes = new Dictionary <long, long>(); completasks = new List <string>(); startedUnwrap = new Dictionary <string, bool>(); if (start > end) { return; } //CreateRange(start, end); //CreateRange2(); tasks = CreateRangeLinks(start, end); //ListGoesRelative(file); ListsGoesRelative(file); StartThreads(5); while (!starter) { } for (int i = 0; i < 2; i++) { UnWrapper unwrapper = new UnWrapper(); System.Threading.Thread unwrapStream = new System.Threading.Thread(unwrapper.UnWrapperStream); unwrapStream.Start(); } while (!wrapstart) { } // CreateResultTable(); Parser parser = new Parser(); //parser.DoParsing2(); //parser.DoParsing_meanDomain2("ru"); parser.DoParsing_meanDomain2SpeedUpTakeFull("ru", "fast2015_5_16_4"); }
List <String> CreateRangeLinks(ArticleDate start, ArticleDate end) { pages = new List <string>(); years = new List <string>(); months = new List <string>(); DateTime cursor = new DateTime(start.time.Ticks); downloaded = new Dictionary <string, bool>(); unwrapped = new Dictionary <string, bool>(); startedDownload = new Dictionary <string, bool>(); synch = new List <bool>(); synch.Add(false); synch.Add(false); List <String> links = new List <String>(); while (cursor.Ticks <= end.time.Ticks) { String line = bas; years.Add(cursor.Year.ToString()); months.Add(cursor.Month.ToString()); //if (cursor.Year < 2010) line += "0"; line += cursor.Year.ToString() + "/" + cursor.Year.ToString() + "-"; if (cursor.Month < 10) { line += "0"; } line += cursor.Month.ToString() + "/"; pages.Add(line); cursor = PlusMonth(cursor); } /*update*/ WebClient w = new WebClient(); foreach (String pagee in pages) { String s = w.DownloadString(pagee); foreach (LinkItem li in LinkFinder.Find(s)) { if (li.Href[0] != 'p') { continue; } links.Add(pagee + li.Href); } } return(links); }