private HtmlNodeNavigator(HtmlNodeNavigator nav)
{
if (nav == null)
{
throw new ArgumentNullException("nav");
}
InternalTrace(null);
_doc = nav._doc;
_currentnode = nav._currentnode;
_attindex = nav._attindex;
_nametable = nav._nametable; // REVIEW: should we do this?
}
internal HtmlNodeNavigator(HtmlDocument doc, HtmlNode currentNode)
{
if (currentNode == null)
{
throw new ArgumentNullException("currentNode");
}
if (currentNode.OwnerDocument != doc)
{
throw new ArgumentException(HtmlDocument.HtmlExceptionRefNotChild);
}
InternalTrace(null);
_doc = doc;
Reset();
_currentnode = currentNode;
}
/// <summary>
/// Begins the process of downloading an internet resource
/// </summary>
/// <param name="uri">Url to the html document</param>
/// <param name="encoding">The encoding to use while downloading the document</param>
/// <param name="credentials">The credentials to use for authenticating the web request</param>
public async Task<HtmlDocument> LoadFromWebAsync(Uri uri, Encoding encoding, NetworkCredential credentials)
{
var clientHandler = new HttpClientHandler();
if (credentials == null)
clientHandler.UseDefaultCredentials = true;
else
clientHandler.Credentials = credentials;
var client = new HttpClient(clientHandler);
var e = await client.GetAsync(uri);
if (e.StatusCode == HttpStatusCode.OK)
{
var html = string.Empty;
if (encoding != null)
{
using (var sr = new StreamReader(await e.Content.ReadAsStreamAsync(), encoding))
{
html = sr.ReadToEnd();
}
}
else
html = await e.Content.ReadAsStringAsync();
var doc = new HtmlDocument();
if (PreHandleDocument != null)
PreHandleDocument(doc);
doc.LoadHtml(html);
return doc;
}
throw new Exception("Error downloading html");
}
private HtmlDocument LoadUrl(Uri uri, string method, IWebProxy proxy, ICredentials creds)
{
HtmlDocument doc = new HtmlDocument();
doc.OptionAutoCloseOnEnd = false;
doc.OptionFixNestedTags = true;
_statusCode = Get(uri, method, null, doc, proxy, creds);
if (_statusCode == HttpStatusCode.NotModified)
{
// read cached encoding
doc.DetectEncodingAndLoad(GetCachePath(uri));
}
return doc;
}
private HttpStatusCode Get(Uri uri, string method, string path, HtmlDocument doc, IWebProxy proxy,
ICredentials creds)
{
string cachePath = null;
bool oldFile = false;
HttpStatusCode status;
using (var request = new HttpRequestMessage(new HttpMethod(method), uri))
using (var handler = new HttpClientHandler())
using (var client = new HttpClient(handler))
{
client.DefaultRequestHeaders.Add("User-Agent", UserAgent);
if (proxy != null)
{
if (creds != null)
{
proxy.Credentials = creds;
handler.Credentials = creds;
}
else
{
proxy.Credentials = CredentialCache.DefaultCredentials;
handler.Credentials = CredentialCache.DefaultCredentials;
}
handler.Proxy = proxy;
handler.UseProxy = true;
}
_fromCache = false;
_requestDuration = 0;
int tc = Environment.TickCount;
if (UsingCache)
{
cachePath = GetCachePath(request.RequestUri);
if (File.Exists(cachePath))
{
client.DefaultRequestHeaders.IfModifiedSince = File.GetLastAccessTime(cachePath);
oldFile = true;
}
}
if (_cacheOnly)
{
if (!File.Exists(cachePath))
{
throw new HtmlWebException("File was not found at cache path: '" + cachePath + "'");
}
if (path != null)
{
IOLibrary.CopyAlways(cachePath, path);
// touch the file
if (cachePath != null) File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
}
_fromCache = true;
return HttpStatusCode.NotModified;
}
if (_useCookies)
{
handler.CookieContainer = new CookieContainer();
}
if (PreRequest != null)
{
// allow our user to change the request at will
if (!PreRequest(handler, request))
{
return HttpStatusCode.ResetContent;
}
// dump cookie
// if (_useCookies)
// {
// foreach(Cookie cookie in req.CookieContainer.GetCookies(req.RequestUri))
// {
// HtmlLibrary.Trace("Cookie " + cookie.Name + "=" + cookie.Value + " path=" + cookie.Path + " domain=" + cookie.Domain);
// }
// }
}
HttpResponseMessage response;
try
{
response = client.SendAsync(request).Result;
}
catch (HttpRequestException)
{
_requestDuration = Environment.TickCount - tc;
if (oldFile)
{
if (path != null)
{
IOLibrary.CopyAlways(cachePath, path);
// touch the file
File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
}
return HttpStatusCode.NotModified;
}
throw;
}
catch (Exception)
{
_requestDuration = Environment.TickCount - tc;
throw;
}
// allow our user to get some info from the response
if (PostResponse != null)
{
PostResponse(request, response);
}
_requestDuration = Environment.TickCount - tc;
_responseUri = response.RequestMessage.RequestUri;
bool html = IsHtmlContent(response.Content.Headers.ContentType.MediaType);
var encoding = response.Content.Headers.ContentEncoding.FirstOrDefault();
Encoding respenc = !string.IsNullOrEmpty(encoding)
? Encoding.GetEncoding(encoding)
: null;
if (response.StatusCode == HttpStatusCode.NotModified)
{
if (UsingCache)
{
_fromCache = true;
if (path != null)
{
IOLibrary.CopyAlways(cachePath, path);
// touch the file
File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
}
return response.StatusCode;
}
// this should *never* happen...
throw new HtmlWebException("Server has send a NotModifed code, without cache enabled.");
}
Stream s = response.Content.ReadAsStreamAsync().Result;
if (s != null)
{
if (UsingCache)
{
// NOTE: LastModified does not contain milliseconds, so we remove them to the file
SaveStream(s, cachePath, RemoveMilliseconds(response.Content.Headers.LastModified), _streamBufferSize);
// save headers
SaveCacheHeaders(request.RequestUri, response);
if (path != null)
{
// copy and touch the file
IOLibrary.CopyAlways(cachePath, path);
File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
}
}
else
{
// try to work in-memory
if ((doc != null) && (html))
{
if (respenc != null)
{
doc.Load(s, respenc);
}
else
{
doc.Load(s, true);
}
}
}
}
status = response.StatusCode;
}
return status;
}
private HttpStatusCode Get(Uri uri, string method, string path, HtmlDocument doc, IWebProxy proxy,
ICredentials creds)
{
string cachePath = null;
HttpWebRequest req;
bool oldFile = false;
req = WebRequest.Create(uri) as HttpWebRequest;
req.Method = method;
req.UserAgent = UserAgent;
if (proxy != null)
{
if (creds != null)
{
proxy.Credentials = creds;
req.Credentials = creds;
}
else
{
proxy.Credentials = CredentialCache.DefaultCredentials;
req.Credentials = CredentialCache.DefaultCredentials;
}
req.Proxy = proxy;
}
_fromCache = false;
_requestDuration = 0;
int tc = Environment.TickCount;
if (UsingCache)
{
cachePath = GetCachePath(req.RequestUri);
if (File.Exists(cachePath))
{
req.IfModifiedSince = File.GetLastAccessTime(cachePath);
oldFile = true;
}
}
if (_cacheOnly)
{
if (!File.Exists(cachePath))
{
throw new HtmlWebException("File was not found at cache path: '" + cachePath + "'");
}
if (path != null)
{
IOLibrary.CopyAlways(cachePath, path);
// touch the file
if (cachePath != null) File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
}
_fromCache = true;
return HttpStatusCode.NotModified;
}
if (_useCookies)
{
req.CookieContainer = new CookieContainer();
}
if (PreRequest != null)
{
// allow our user to change the request at will
if (!PreRequest(req))
{
return HttpStatusCode.ResetContent;
}
// dump cookie
// if (_useCookies)
// {
// foreach(Cookie cookie in req.CookieContainer.GetCookies(req.RequestUri))
// {
// HtmlLibrary.Trace("Cookie " + cookie.Name + "=" + cookie.Value + " path=" + cookie.Path + " domain=" + cookie.Domain);
// }
// }
}
HttpWebResponse resp;
try
{
resp = req.GetResponse() as HttpWebResponse;
}
catch (WebException we)
{
_requestDuration = Environment.TickCount - tc;
resp = (HttpWebResponse)we.Response;
if (resp == null)
{
if (oldFile)
{
if (path != null)
{
IOLibrary.CopyAlways(cachePath, path);
// touch the file
File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
}
return HttpStatusCode.NotModified;
}
throw;
}
}
catch (Exception)
{
_requestDuration = Environment.TickCount - tc;
throw;
}
// allow our user to get some info from the response
if (PostResponse != null)
{
PostResponse(req, resp);
}
_requestDuration = Environment.TickCount - tc;
_responseUri = resp.ResponseUri;
bool html = IsHtmlContent(resp.ContentType);
Encoding respenc = !string.IsNullOrEmpty(resp.ContentEncoding)
? Encoding.GetEncoding(resp.ContentEncoding)
: null;
if (OverrideEncoding != null)
respenc = OverrideEncoding;
if (resp.StatusCode == HttpStatusCode.NotModified)
{
if (UsingCache)
{
_fromCache = true;
if (path != null)
{
IOLibrary.CopyAlways(cachePath, path);
// touch the file
File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
}
return resp.StatusCode;
}
// this should *never* happen...
throw new HtmlWebException("Server has send a NotModifed code, without cache enabled.");
}
Stream s = resp.GetResponseStream();
if (s != null)
{
if (UsingCache)
{
// NOTE: LastModified does not contain milliseconds, so we remove them to the file
SaveStream(s, cachePath, RemoveMilliseconds(resp.LastModified), _streamBufferSize);
// save headers
SaveCacheHeaders(req.RequestUri, resp);
if (path != null)
{
// copy and touch the file
IOLibrary.CopyAlways(cachePath, path);
File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
}
}
else
{
// try to work in-memory
if (doc != null && html)
{
if (respenc == null)
{
doc.Load(s, true);
}
else
{
doc.Load(s, respenc);
}
}
}
resp.Close();
}
return resp.StatusCode;
}
/// <summary>
/// Loads an HTML document from an Internet resource.
/// </summary>
/// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
/// <param name="method">The HTTP method used to open the connection, such as GET, POST, PUT, or PROPFIND.</param>
/// <param name="proxy">Proxy to use with this request</param>
/// <param name="credentials">Credentials to use when authenticating</param>
/// <returns>A new HTML document.</returns>
public HtmlDocument Load(string url, string method, IWebProxy proxy, ICredentials credentials)
{
Uri uri = new Uri(url);
HtmlDocument doc;
#if NET20 || NET40 || NET451
if (uri.Scheme == Uri.UriSchemeFile)
#else
// Uri.UriScheme* is internal. Bug or not completed?
if (uri.Scheme == "file")
#endif
{
doc = LoadUrl(uri, method, proxy, credentials);
}
else
{
#if NET20 || NET40 || NET451
if (uri.Scheme == Uri.UriSchemeFile)
#else
// Uri.UriScheme* is internal. Bug or not completed?
if (uri.Scheme == "file")
#endif
{
doc = new HtmlDocument();
doc.OptionAutoCloseOnEnd = false;
doc.OptionAutoCloseOnEnd = true;
doc.DetectEncodingAndLoad(url, _autoDetectEncoding);
}
else
{
throw new HtmlWebException("Unsupported uri scheme: '" + uri.Scheme + "'.");
}
}
if (PreHandleDocument != null)
{
PreHandleDocument(doc);
}
return doc;
}
/// <summary>
/// Loads an HTML document from an Internet resource.
/// </summary>
/// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
/// <param name="method">The HTTP method used to open the connection, such as GET, POST, PUT, or PROPFIND.</param>
/// <returns>A new HTML document.</returns>
public HtmlDocument Load(string url, string method)
{
Uri uri = new Uri(url);
HtmlDocument doc;
#if NET20 || NET40 || NET451
if ((uri.Scheme == Uri.UriSchemeHttps) ||
(uri.Scheme == Uri.UriSchemeHttp))
#else
// Uri.UriSchemeHttps is internal. Bug or not completed?
if ((uri.Scheme == "https") ||
(uri.Scheme == "http"))
#endif
{
doc = LoadUrl(uri, method, null, null);
}
else
{
#if NET20 || NET40 || NET451
if (uri.Scheme == Uri.UriSchemeFile)
#else
// Uri.UriScheme* is internal. Bug or not completed?
if (uri.Scheme == "file")
#endif
{
doc = new HtmlDocument();
doc.OptionAutoCloseOnEnd = false;
doc.OptionAutoCloseOnEnd = true;
if (OverrideEncoding != null)
doc.Load(url, OverrideEncoding);
else
doc.DetectEncodingAndLoad(url, _autoDetectEncoding);
}
else
{
throw new HtmlWebException("Unsupported uri scheme: '" + uri.Scheme + "'.");
}
}
if (PreHandleDocument != null)
{
PreHandleDocument(doc);
}
return doc;
}
internal HtmlAttribute(HtmlDocument ownerdocument)
{
_ownerdocument = ownerdocument;
}
/// <summary>
/// Helper function that returns an HTML document from text
/// </summary>
private static HtmlDocument GetHtml(string source)
{
HtmlDocument html = new HtmlDocument();
html.OptionFixNestedTags = true;
html.OptionAutoCloseOnEnd = true;
html.OptionDefaultStreamEncoding = Encoding.UTF8;
html.LoadHtml(source ?? "");
// Encode any code blocks independently so they won't
// be stripped out completely when we do a final cleanup
foreach (var n in html.DocumentNode.DescendantsAndSelf())
{
if (n.Name == "code")
{
//** Code tag attribute vulnerability fix 28-9-12 (thanks to Natd)
HtmlAttribute[] attr = n.Attributes.ToArray();
foreach (HtmlAttribute a in attr)
{
if (a.Name != "style" && a.Name != "class") { a.Remove(); }
} //** End fix
n.InnerHtml = System.Net.WebUtility.HtmlEncode(System.Net.WebUtility.HtmlDecode(n.InnerHtml));
}
}
return html;
}
/// <summary>
/// Creates an HTML node from a string representing literal HTML.
/// </summary>
/// <param name="html">The HTML text.</param>
/// <returns>The newly created node instance.</returns>
public static HtmlNode CreateNode(string html)
{
// REVIEW: this is *not* optimum...
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
return doc.DocumentNode.FirstChild;
}
/// <summary>
/// Initializes HtmlNode, providing type, owner and where it exists in a collection
/// </summary>
/// <param name="type"></param>
/// <param name="ownerdocument"></param>
/// <param name="index"></param>
public HtmlNode(HtmlNodeType type, HtmlDocument ownerdocument, int index)
{
_nodetype = type;
_ownerdocument = ownerdocument;
_outerstartindex = index;
switch (type)
{
case HtmlNodeType.Comment:
Name = HtmlNodeTypeNameComment;
_endnode = this;
break;
case HtmlNodeType.Document:
Name = HtmlNodeTypeNameDocument;
_endnode = this;
break;
case HtmlNodeType.Text:
Name = HtmlNodeTypeNameText;
_endnode = this;
break;
}
if (_ownerdocument.Openednodes != null)
{
if (!Closed)
{
// we use the index as the key
// -1 means the node comes from public
if (-1 != index)
{
_ownerdocument.Openednodes.Add(index, this);
}
}
}
if ((-1 != index) || (type == HtmlNodeType.Comment) || (type == HtmlNodeType.Text)) return;
// innerhtml and outerhtml must be calculated
SetChanged();
}
