List <String> CreateRangeLinks(ArticleDate start, ArticleDate end)
{
pages = new List <string>();
years = new List <string>();
months = new List <string>();
DateTime cursor = new DateTime(start.time.Ticks);
downloaded = new Dictionary <string, bool>();
unwrapped = new Dictionary <string, bool>();
startedDownload = new Dictionary <string, bool>();
synch = new List <bool>();
synch.Add(false);
synch.Add(false);
List <String> links = new List <String>();
while (cursor.Ticks <= end.time.Ticks)
{
String line = bas;
years.Add(cursor.Year.ToString());
months.Add(cursor.Month.ToString());
//if (cursor.Year < 2010) line += "0";
line += cursor.Year.ToString() + "/" + cursor.Year.ToString() + "-";
if (cursor.Month < 10)
{
line += "0";
}
line += cursor.Month.ToString() + "/";
pages.Add(line);
cursor = PlusMonth(cursor);
}
/*update*/
WebClient w = new WebClient();
foreach (String pagee in pages)
{
String s = w.DownloadString(pagee);
foreach (LinkItem li in LinkFinder.Find(s))
{
if (li.Href[0] != 'p')
{
continue;
}
links.Add(pagee + li.Href);
}
}
return(links);
}
public void DownloadStreamUpdate()
{
Console.WriteLine("Downloader started! My number is " + mynumber.ToString());
// for (int i = 0; i < pages.Count; i++)
foreach (String pagee in pages)
{
WebClient w = new WebClient();
while (BunchHelper.outFile > 4)
{
;
}
while (BunchHelper.rawFile > 4)
{
;
}
String s = w.DownloadString(pagee);
Int16 day = 0;
Boolean flag = true;
Int16 counter = -1;
char[] delimiterChars = { '-', ' ' };
Int32 index = -1;
foreach (LinkItem li in LinkFinder.Find(s))
{
if (li.Href[0] != 'p')
{
continue;
}
if (day == 1 && flag)
{
flag = false;
continue;
}
;
counter++;
String[] check = li.Text.Split(delimiterChars);
Int16 validationHour = Int16.Parse(check[2].Substring(0, 2));
if (counter != validationHour)
{
counter = validationHour;
}
if (counter == 24)
{
counter = 0;
}
if (counter != 19)
{
continue;
}
String address = pagee + li.Href;
index++;
String to = @"d:\wp\code\3_wikiParserRealDigger\wikiParserRealDigger\bin\Debug\" + li.Href;// "_" + index.ToString();
day++;
if (BunchHelper.startedDownload.ContainsKey(li.Href))
{
continue;
}
BunchHelper.startedDownload.Add(li.Href, true);
using (var client = new WebClient())
{
BunchHelper.rawFile++;
// while (BunchHelper.rawFile >= 3)
// { ;}
Console.WriteLine(" I am downloading " + li.Href);
trymore:
try { client.DownloadFile(address, to); }
catch (System.Net.WebException ex) { System.Threading.Thread.Sleep(10000); goto trymore; }
}
BunchHelper.downloaded.Add(li.Href, true);
BunchHelper.starter = true;
}
}
Console.WriteLine("Downloader ended");
BunchHelper.downloadThreadEnd = true;
}