private void DownloadLink(Link l)
{
try
{
curUri_ = l.URL;
NotifyPropertyChanged("CurrentUri");
CookieContainer localCookies = MakeLocalCookies(l.URL);
HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(l.URL);
FillHTTPWebRequest(hwr, l, localCookies);
using (HttpWebResponse hwrs = (HttpWebResponse)hwr.GetResponse())
{
lock (Workers.cookies)
{
Workers.cookies.Add(hwrs.Cookies);
}
switch (l.Action)
{
case LinkAction.Download:
Writer.Receive(this,hwrs, l);
break;
case LinkAction.Parse:
Parser.Receive(this,hwrs, l);
break;
}
}
}
catch (UriFormatException e)
{
l.Error = e.Message;
p.BadLinks.Offer(l);
}
catch (WebException e)
{
l.Error = e.Message;
p.BadLinks.Offer(l);
}
catch (Exception e)
{
l.Error = e.Message;
p.BadLinks.Offer(l);
}
curUri_ = null;
NotifyPropertyChanged("CurrentUri");
Progress.Reset();
}
public void Receive(Worker w, HttpWebResponse hwr, Link l)
{
char[] seps = {'/'};
Regex badpath = new Regex(String.Format("[{0}]", Regex.Escape(new String(Path.GetInvalidPathChars()))));
Regex badfn = new Regex(String.Format("[{0}]", Regex.Escape(new String(Path.GetInvalidFileNameChars()))));
String destdir = badpath.Replace(l.DestDir, "_");
String filename = badfn.Replace(l.FileName, "_");
String dir = Path.Combine(p.RootDirectory,Path.Combine(destdir.Split(seps)));
String file = Path.Combine(dir, filename);
if (!Directory.Exists(dir))
Directory.CreateDirectory(dir);
if (!p.CanWrite(file, hwr.LastModified))
return;
String tmpfile = file + ".dlprog_part";
using (FileStream fs = File.Open(tmpfile, FileMode.Create))
{
Stream s = hwr.GetResponseStream();
//s.CopyTo(fs);
const int BUFFERSIZE = 8192;
w.Progress.BytesTotal = hwr.ContentLength;
byte[] buf = new byte[BUFFERSIZE];
int bytesread = 0;
while ((bytesread = s.Read(buf, 0, BUFFERSIZE)) != 0)
{
fs.Write(buf, 0, bytesread);
w.Progress.BytesDownloaded += bytesread;
}
}
File.Move(tmpfile, file);
}
protected override void Parse(String html, Link l, DateTime lastModified)
{
try
{
curLink = l;
lua["page"] = new PageInterfaceToLua { HTML = html, URL = l.URL.OriginalString, UserData = l.UserData };
lastModified_ = lastModified;
lua.DoString(p.Script);
}
catch (LuaException e)
{
throw new ApplicationException("Lua error: " + e.Message);
}
}
private void FillHTTPWebRequest(HttpWebRequest hwr, Link l, CookieContainer cookies)
{
hwr.CookieContainer = cookies;
hwr.Method = l.HTTPMethod;
hwr.ContentType = l.ContentType;
hwr.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0";
hwr.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
hwr.Headers.Add("Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7");
hwr.Headers.Add("Accept-Language: nl, en-gb;q=0.9, en;q=0.8");
if (l.Body != null && l.Body != "")
{
Stream s = hwr.GetRequestStream();
using (StreamWriter sw = new StreamWriter(s))
{
sw.Write(l.Body);
}
}
}
protected abstract void Parse(String html, Link l, DateTime lastModified);
public void Receive(Worker w, HttpWebResponse hwr, Link l)
{
StreamReader sr = new StreamReader(hwr.GetResponseStream());
Parse(sr.ReadToEnd(), l, hwr.LastModified);
sr.Close();
}
