Result Download()
{
HttpWebRequest request = WebRequest.CreateHttp(org_url.str);
request.Method = "GET";
request.Accept = "*/*";
request.UserAgent = data.user_agent;
request.AllowAutoRedirect = true;
request.MaximumAutomaticRedirections = 10;
request.Timeout = 5000;
data.Log("--->ASSET Descargando: " + org_url.str);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
final_url = new URL(response.ResponseUri.OriginalString);
StatusReport report = data.CheckAsset(final_url);
switch (report.url_status)
{
case UrlStatus.Failed:
return(Result.Fail | Result.Exists);
case UrlStatus.Free:
data.UpdateStatus(this, UrlStatus.Iprg);
if (!MakeFullPath())
{
return(Result.Exists);
}
Stream rs = response.GetResponseStream();
if (in_string)
{
str_resp = new StreamReader(rs).ReadToEnd();
File.WriteAllText(file, str_resp);
}
else
{
FileStream fs = File.OpenWrite(file);
byte[] buffer = new byte[response.ContentLength];
long copied = 0;
//TODO Make this bufer HD I/O friendly
while (copied < response.ContentLength)
{
int readed = rs.Read(buffer, 0, buffer.Length);
fs.Write(buffer, 0, readed);
copied += readed;
}
fs.Close();
}
response.Close();
return(Result.Ok);
case UrlStatus.Iprg:
case UrlStatus.Saved:
return(Result.Exists);
default:
return(Result.Exists);
}
}
void LinkProc(HtmlNode node)
{
string link = node.GetAttributeValue("href", null);
if (link != null)
{
link.Trim();
}
else
{
return;
}
//Links Regex
foreach (string rgx in data.links_regex)
{
if (Regex.IsMatch(link, rgx))
{
return;
}
}
URL url_link;
if (URL.IsRelative(link))
{
url_link = new URL(page.final_url.str, link);
}
else
{
url_link = new URL(link);
}
//Its same page?
if (link == page.final_url.url_main.file)
{
node.SetAttributeValue("href", page.filename);
return;
}
//Its internal?
if (link.StartsWith("#"))
{
return;
}
//Nav Direction
URL.NavType nt = url_link.Compare(data.org_links[page.org_link]);
if (nt == URL.NavType.Same)
{
StatusReport sr = data.CheckLink(url_link);
if (sr.page != null)
{
node.SetAttributeValue("href", sr.page.PathFromRelative(page.final_url));
}
return;
}
switch (data.nav_direction)
{
case NavDirection.In:
if (nt != URL.NavType.In &&
!(data.hnav && nt == URL.NavType.Side))
{
return;
}
break;
case NavDirection.Out:
if (nt != URL.NavType.Out &&
!(data.hnav && nt == URL.NavType.Side))
{
return;
}
break;
case NavDirection.Static:
if (nt != URL.NavType.Side)
{
return;
}
break;
case NavDirection.Both:
if (!data.hnav && nt == URL.NavType.Side)
{
break;
}
return;
}
//Its already dowloaded?
StatusReport report = data.CheckLink(url_link);
//FOR DOWNLOAD
if (report.url_status == UrlStatus.Free)
{
data.Log("-->PROC: Link Pending");
Page link_page = new Page(ref data, url_link, page.org_link);
link_page.Proccess();
switch (link_page.result)
{
case Result.Ok:
string final_link = link_page.PathFromRelative(page.final_url);
node.SetAttributeValue("href", final_link);
data.UpdateStatus(link_page, UrlStatus.ToDo);
break;
case Result.IsAsset:
//TODO Do something
break;
case Result.Fail:
data.UpdateStatus(link_page, UrlStatus.Failed);
break;
}
//DOWNLOADED
}
else if (report.url_status != UrlStatus.Failed)
{
//TODO
string final_link = report.page.PathFromRelative(page.final_url);
node.SetAttributeValue("href", final_link);
}
}
public Asset(URL url, ref MainData main_data)
{
data = main_data;
org_url = url;
}
string AssetProc(string src, Type type, string rel = null)
{
if (src == null)
{
return(null);
}
if (rel == null)
{
rel = page.final_url.str;
}
URL url_src;
if (URL.IsRelative(src))
{
url_src = new URL(rel, src);
}
else
{
url_src = new URL(src);
}
StatusReport report = data.CheckAsset(url_src);
if (report.url_status == UrlStatus.Iprg || report.url_status == UrlStatus.Saved)
{
return(report.asset.final_url.RelativeTo(rel).org_str);
}
else if (report.url_status == UrlStatus.Free)
{
Asset asset = new Asset(url_src, ref data);
if (type == Type.css)
{
asset.Process(true);
}
else
{
asset.Process();
}
if (asset.result == Result.Ok || asset.result == Result.Exists)
{
if (asset.result == Result.Ok)
{
data.UpdateStatus(asset, UrlStatus.Saved);
if (type == Type.css)
{
css_todo.Add(asset);
}
}
return(asset.final_url.RelativeTo(rel).org_str);
}
else if (asset.result == Result.Fail)
{
data.UpdateStatus(asset, UrlStatus.Failed);
return(null);
}
}
return(null);
}