void CreateRange(ArticleDate start, ArticleDate end)
{
pages = new List <string>();
years = new List <string>();
months = new List <string>();
DateTime cursor = new DateTime(start.time.Ticks);
downloaded = new Dictionary <string, bool>();
unwrapped = new Dictionary <string, bool>();
startedDownload = new Dictionary <string, bool>();
synch = new List <bool>();
synch.Add(false);
synch.Add(false);
while (cursor.Ticks <= end.time.Ticks)
{
String line = bas;
years.Add(cursor.Year.ToString());
months.Add(cursor.Month.ToString());
//if (cursor.Year < 2010) line += "0";
line += cursor.Year.ToString() + "/" + cursor.Year.ToString() + "-";
if (cursor.Month < 10)
{
line += "0";
}
line += cursor.Month.ToString() + "/";
pages.Add(line);
cursor = PlusMonth(cursor);
}
}
public void processPipeFile(String[] file, String dd = "")
{
try
{
RulesDictionary.CreateSelectedList(@"D:\wp\pipe\treatmentinput\treatmentgroups.txt");
}
catch (Exception ex)
{
}
start_ = DateTime.ParseExact(Start, "MM/yyyy", null);
end_ = DateTime.ParseExact(End, "MM/yyyy", null);
start = new ArticleDate(DateTime.ParseExact(Start, "MM/yyyy", null));
end = new ArticleDate(DateTime.ParseExact(End, "MM/yyyy", null));
borns = new Dictionary <Int64, DateTime>();
pagesizes = new Dictionary <long, long>();
completasks = new List <string>();
startedUnwrap = new Dictionary <string, bool>();
if (start > end)
{
return;
}
//CreateRange(start, end);
//CreateRange2();
tasks = CreateRangeLinks(start, end);
//ListGoesRelative(file);
ListsGoesRelative(file);
StartThreads(5);
while (!starter)
{
}
for (int i = 0; i < 2; i++)
{
UnWrapper unwrapper = new UnWrapper();
System.Threading.Thread unwrapStream = new System.Threading.Thread(unwrapper.UnWrapperStream);
unwrapStream.Start();
}
while (!wrapstart)
{
}
//
CreateResultTable();
Parser parser = new Parser();
//parser.DoParsing2();
//parser.DoParsing_meanDomain2("ru");
parser.DoParsing_meanDomain2SpeedUpTakeFull("ru", "fast2015_5_16_4");
}
List <String> CreateRangeLinks(ArticleDate start, ArticleDate end)
{
pages = new List <string>();
years = new List <string>();
months = new List <string>();
DateTime cursor = new DateTime(start.time.Ticks);
downloaded = new Dictionary <string, bool>();
unwrapped = new Dictionary <string, bool>();
startedDownload = new Dictionary <string, bool>();
synch = new List <bool>();
synch.Add(false);
synch.Add(false);
List <String> links = new List <String>();
while (cursor.Ticks <= end.time.Ticks)
{
String line = bas;
years.Add(cursor.Year.ToString());
months.Add(cursor.Month.ToString());
//if (cursor.Year < 2010) line += "0";
line += cursor.Year.ToString() + "/" + cursor.Year.ToString() + "-";
if (cursor.Month < 10)
{
line += "0";
}
line += cursor.Month.ToString() + "/";
pages.Add(line);
cursor = PlusMonth(cursor);
}
/*update*/
WebClient w = new WebClient();
foreach (String pagee in pages)
{
String s = w.DownloadString(pagee);
foreach (LinkItem li in LinkFinder.Find(s))
{
if (li.Href[0] != 'p')
{
continue;
}
links.Add(pagee + li.Href);
}
}
return(links);
}