public override SPCoder.Utils.Nodes.BaseNode GetSPStructure(string siteUrl) { BaseNode rootNode = new PageNode(); HtmlWeb page = new HtmlWeb(); HtmlAgilityPack.HtmlWeb.PreRequestHandler handler = delegate(HttpWebRequest request) { request.Headers[HttpRequestHeader.AcceptEncoding] = "gzip, deflate"; request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip; request.CookieContainer = new System.Net.CookieContainer(); return(true); }; page.PreRequest += handler; Document = page.Load(siteUrl); page.Get(siteUrl, "/"); rootNode.Title = RootNodeTitle + page.ResponseUri.Host.ToString(); rootNode.IconPath = "html.png"; BaseNode my = new PageNode(Document.DocumentNode); my.RootNode = rootNode; my.ParentNode = rootNode; my.Title = Document.DocumentNode.Name; my.SPObject = Document; //return rootNode; //rootNode.SPObject = site; doPageNodes(Document.DocumentNode, rootNode, rootNode); return(rootNode); }
private void btnBuscaCEP_Click(int row) { string url = "https://www.mapacep.com.br//busca-cep.php?busca-cep=" + dataGridView1.Rows[row].Cells[15].Value.ToString(); HtmlWeb page = new HtmlWeb(); HtmlAgilityPack.HtmlDocument document = page.Load(url); page.Get(url, "/"); document.Save("out.html"); var cepAddress = document.DocumentNode.Descendants("title").FirstOrDefault().InnerText.Split(','); try { address.state = cepAddress[3].Split(' ')[2].Trim(); address.city = cepAddress[2].Split(' ')[2].Trim(); address.district = cepAddress[1].Trim(); address.street = cepAddress[0].Split('-')[0].Trim(); address.lat = double.Parse(cepAddress[6].Split(' ')[2]); address.longitude = double.Parse(cepAddress[7].Split(' ')[2]); } catch { } }
public void SaveSnapshot(Uri uri) { var urlString = uri.ToString(); // This folder will be created before download process will copy files into it var folderPath = _storageProvider.Combine("_ArchivedLinks", urlString.GetHashCode().ToString()); var contentType = GetContentType(uri); var allowableFileContentTypes = new string[] { "application/pdf", "image/jpeg", "image/jpg", "image/png", "image/gif" }; if (allowableFileContentTypes.Contains(contentType)) { DownloadFile(urlString, uri, folderPath, true); } else if (contentType == "text/html") { var htmlWeb = new HtmlWeb(); var document = htmlWeb.Load(urlString); DownloadHtml(ref document, uri, folderPath); htmlWeb.Get(urlString, "/"); var indexPath = _storageProvider.Combine(folderPath, "index.html"); if (_storageProvider.FileExists(indexPath)) { _storageProvider.DeleteFile(indexPath); } using (var stream = _storageProvider.CreateFile(indexPath).OpenWrite()) { document.Save(stream); } } else { throw new NotSupportedException("Uri type not supported"); } }