public IWebPageReader OnClickCallback(WebPageData item) { string content = (string)item.Tag; simulatedPage.content = content; return(simulatedPage); }
public WebPageData[] ExtractData(HtmlDocument doc) { List <WebPageData> data = new List <WebPageData>(); var container = HtmlNode.CreateNode(content); var mainNode = Helper.AnyChild(container, "b"); if (mainNode != null) { var aLinkNodes = Helper.AllChild(container, "a"); if (aLinkNodes == null) { return(null); } foreach (var alinkNode in aLinkNodes) { string link = ""; if (alinkNode.HasAttributes) { link = alinkNode.GetAttributeValue("href", ""); } link = link.Contains(webFolder) ? "" : webFolder + link; WebPageData singleData = WebPageData.GetTextOnly($"{alinkNode.InnerText} - {mainNode.InnerText.Replace(':', '\0')}", ""); singleData.IsFinal = true; //singleData.underlayingLinkReader = new IdlebrainAlbumPageReader(link);//TODO: MEMORY LEAK -> don't create new instance for every item singleData.Tag = link; //add the link in tag data.Add(singleData); } return(data.ToArray()); } return(null); }
public IWebPageReader OnClickCallback(WebPageData item) { var link = (string)item.Tag; albumPageReader.ChangeUrl(link); return(albumPageReader); }
/// <summary> /// DBインサート /// </summary> public void Ins(WebPageData wpd) { try { var dbConnection = new SQLiteConnect(); using (dbConnection.conn) using (var transaction = dbConnection.conn.BeginTransaction()) { var cmd = dbConnection.conn.CreateCommand(); cmd.CommandText = "REPLACE INTO pages (title, url, date) VALUES (@TITLE, @URL, @DATE)"; cmd.Parameters.Add(SetParam("@TITLE", DbType.String, wpd.Title)); cmd.Parameters.Add(SetParam("@URL", DbType.String, wpd.URL)); cmd.Parameters.Add(SetParam("@DATE", DbType.String, wpd.nowtime.ToString())); cmd.ExecuteNonQuery(); transaction.Commit(); } } catch (Exception ex) { throw ex; } }
public WebPageData[] ExtractData(HtmlDocument doc) { if (doc == null && cached != null) { return(cached); //return data from cache } if (doc == null) { return(null); //error protection } Dictionary <string, string> att = new Dictionary <string, string>(); att.Add("align", "left"); try { var container = Helper.AllChild(doc.DocumentNode, "div", att, true)[1]; if (container == null) { return(null); } //container = Helper.AnyChild(container, "p"); string innHtml = container.InnerHtml; string[] stElements = innHtml.Split(new string[] { "<br>" }, StringSplitOptions.RemoveEmptyEntries); List <WebPageData> data = new List <WebPageData>(); foreach (var st in stElements) { var sss = $"<div>{Helper.TrimToEntry(st)}</div>"; HtmlNode pNode = HtmlNode.CreateNode(sss); var mainNode = Helper.AnyChild(pNode, "b"); if (mainNode != null) { var aLinkNodes = Helper.AllChild(pNode, "a"); var singleData = WebPageData.GetTextOnly( mainNode.InnerText.Replace(':', '\0'), aLinkNodes != null ? $"Contain : {aLinkNodes.Count.ToString()}" : ""); //singleData.underlayingLinkReader = new IdlebrainSimulatedPage(sss);//TODO: MEMORY LEAK -> don't create new instance for every item singleData.Tag = sss; //Add the data as tag singleData.IsFinal = true; data.Add(singleData); } } this.IsSimulation = true; return(cached = data.ToArray()); } catch (Exception) { } return(null); }
public async Task <WebPageData> ScrapWebPageAsync(string url) { var content = await GetPageContentAsync(url); var title = GetPageTitle(content); var text = ParsePageContent(content); var webPageData = new WebPageData(url: url, title: title, content: text); _logger.LogInformation($"Created web page data: \n Id: {webPageData.Id}\n Url: {url} \n Title: {title} \n Content-Length: {text.Length}"); return(webPageData); }
///print web site data function static void PrintWebPageData(string url) { try { WebPageData webPageData = WebPageData.Get(url); Console.WriteLine($"Address:{webPageData.URL},Size:{webPageData.Length},DownloadDuration:{ webPageData.DownloadElapsedMiliSeconds}"); } catch (Exception e) { Console.WriteLine(e.Message); } }
protected void MoveUpBtn_Click(object sender, ImageClickEventArgs e) { HtmlTable Table = (HtmlTable)LoginView1.FindControl("PageDetails"); TreeView pagesTV = (TreeView)LoginView1.FindControl("PagesTreeView"); SiteMapDataSource source = (SiteMapDataSource)LoginView1.FindControl("SiteMapDataSource1"); string temp = pagesTV.SelectedValue; string pageid = WebPageData.GetWebPageId(temp); editor = new SitemapEditor(); editor.MoveUp(pageid); editor.Save(); pagesTV.DataSource = source; pagesTV.DataBind(); }
public WebPageData[] ExtractData(HtmlDocument doc) { AlbumImages.Clear(); Dictionary <string, string> att = new Dictionary <string, string>(); att.Add("width", "100%"); att.Add("style", "background-color: white;"); var container = Helper.AnyChild(doc.DocumentNode, "table", att, true); if (container == null) { return(null); } //TODO : add support for http://www.idlebrain.com/movie/photogallery/madhusharma1.html var imgNodes = Helper.AllChild(container, "img"); if (imgNodes == null) { return(null); } int index = 0; List <WebPageData> data = new List <WebPageData>(); foreach (var imgNode in imgNodes) { string thSrc = (webDir.EndsWith("/") ? webDir : webDir + "/") + imgNode.GetAttributeValue("src", ""); var singleData = WebPageData.GetTextOnly($"{index++}", ""); singleData.IsFinal = true; singleData.UID = UidGenerator(); singleData.ImageUrl = thSrc; //image link var imgDefi = new ImageDefinition() { thumbnil = thSrc, original = (webDir.EndsWith("/") ? webDir : webDir + "/") + imgNode.GetAttributeValue("src", "").Replace("th_", "") }; AlbumImages.Add(imgDefi); data.Add(singleData); } return(data.ToArray()); }
protected void NewPageBtn_Click(object sender, EventArgs e) { HtmlTable Table = (HtmlTable)LoginView1.FindControl("PageDetails"); TreeView pagesTV = (TreeView)LoginView1.FindControl("PagesTreeView"); SiteMapDataSource source = (SiteMapDataSource)LoginView1.FindControl("SiteMapDataSource1"); string pageid = WebPageData.NewWebPage(); WebPage page = WebPageData.LoadPageData(pageid); editor = new SitemapEditor(); editor.InsertPage(page); editor.Save(); WebPageData.UpdateVirtualPath(pageid); pagesTV.DataSource = source; pagesTV.DataBind(); Table.Visible = false; }
public IWebPageReader OnClickCallback(WebPageData item) { switch (currentState) { case CurrentStateInfo.Unknow: break; case CurrentStateInfo.ListOfPerson: AlbumImages.Clear(); if (architecture.ListOfAlbumReader == null) //one level skipped { currentState = CurrentStateInfo.ListOfAlbum; return(OnClickCallback(item)); } architecture.ListOfAlbumReader.Url = architecture.ListOfAlbumReader.GetUrl((Person)item.Tag); LoadConfig(architecture.ListOfAlbumReader); informationForLevel = item.Tag; return(this); case CurrentStateInfo.ListOfAlbum: AlbumImages.Clear(); if (architecture.ListOfAlbumReader == null) { try { item.Tag = Changer.ToAlbum((Person)item.Tag); } catch (Exception) { } } architecture.ListOfImagesReader.Url = architecture.ListOfImagesReader.GetUrl((Album)item.Tag); LoadConfig(architecture.ListOfImagesReader); informationForLevel = item.Tag; return(this); case CurrentStateInfo.ListOfImage: break; case CurrentStateInfo.ShowImage: break; default: break; } return(null); }
protected void UpdatePageBtn_Click(object sender, EventArgs e) { HtmlTable Table = (HtmlTable)LoginView1.FindControl("PageDetails"); Table.Visible = true; TreeView pagesTV = (TreeView)LoginView1.FindControl("PagesTreeView"); SiteMapDataSource source = (SiteMapDataSource)LoginView1.FindControl("SiteMapDataSource1"); TextBox TitleTB = (TextBox)LoginView1.FindControl("TitleTextBox"); TextBox NavigationTB = (TextBox)LoginView1.FindControl("NavigationTextBox"); TextBox VirtualPathTB = (TextBox)LoginView1.FindControl("VirtualPathTextBox"); DropDownList AccessRolesDDL = (DropDownList)LoginView1.FindControl("AccessRolesDropDownList"); DropDownList EditRolesDDL = (DropDownList)LoginView1.FindControl("EditRolesDropDownList"); CheckBox VisibleCB = (CheckBox)LoginView1.FindControl("VisibleCheckBox"); string temp = pagesTV.SelectedValue; string pageid = WebPageData.GetWebPageId(temp); WebPage page = WebPageData.LoadPageData(pageid); page.Title = TitleTB.Text; page.NavigationName = NavigationTB.Text; if ((VirtualPathTB.Text != string.Empty) && (WebPageData.IsValidUrl(VirtualPathTB.Text))) { page.VirtualPath = VirtualPathTB.Text.ToLower(); } else { page.VirtualPath = string.Format("~/default{0}.aspx", pageid); } page.AccessRole = AccessRolesDDL.SelectedValue; page.EditRole = EditRolesDDL.SelectedValue; page.Visible = VisibleCB.Checked; WebPageData.UpdatePage(page); editor = new SitemapEditor(); editor.UpdatePage(page); editor.Save(); pagesTV.DataSource = source; pagesTV.DataBind(); Table.Visible = false; }
protected void PagesTreeView_SelectedNodeChanged(object sender, EventArgs e) { HtmlTable Table = (HtmlTable)LoginView1.FindControl("PageDetails"); Table.Visible = true; TreeView pagesTV = (TreeView)LoginView1.FindControl("PagesTreeView"); SiteMapDataSource source = (SiteMapDataSource)LoginView1.FindControl("SiteMapDataSource1"); TextBox TitleTB = (TextBox)LoginView1.FindControl("TitleTextBox"); TextBox NavigationTB = (TextBox)LoginView1.FindControl("NavigationTextBox"); TextBox VirtualPathTB = (TextBox)LoginView1.FindControl("VirtualPathTextBox"); DropDownList AccessRolesDDL = (DropDownList)LoginView1.FindControl("AccessRolesDropDownList"); DropDownList EditRolesDDL = (DropDownList)LoginView1.FindControl("EditRolesDropDownList"); CheckBox VisibleCB = (CheckBox)LoginView1.FindControl("VisibleCheckBox"); string temp = pagesTV.SelectedValue; if (temp != "Site Map") { string pageid = WebPageData.GetWebPageId(temp); WebPage page = WebPageData.LoadPageData(pageid); TitleTB.Text = page.Title; NavigationTB.Text = page.NavigationName; VirtualPathTB.Text = page.VirtualPath; VisibleCB.Checked = page.Visible; AccessRolesDDL.DataSource = Roles.GetAllRoles(); AccessRolesDDL.DataBind(); AccessRolesDDL.Items.Insert(0, new ListItem("Anonymous", "Anonymous")); AccessRolesDDL.SelectedValue = page.AccessRole; EditRolesDDL.DataSource = Roles.GetAllRoles(); EditRolesDDL.DataBind(); EditRolesDDL.SelectedValue = page.EditRole; } else { Response.Redirect("~/administration/navigation.aspx"); } }
protected void DeletePageBtn_Click(object sender, EventArgs e) { HtmlTable Table = (HtmlTable)LoginView1.FindControl("PageDetails"); TreeView pagesTV = (TreeView)LoginView1.FindControl("PagesTreeView"); SiteMapDataSource source = (SiteMapDataSource)LoginView1.FindControl("SiteMapDataSource1"); if (pagesTV.SelectedValue != string.Empty) { string temp = pagesTV.SelectedValue; string pageid = WebPageData.GetWebPageId(temp); ArrayList list = ModuleData.GetAllModules(pageid); if ((pageid != "1") && (list.Count == 0)) { WebPageData.DeletePage(pageid); editor = new SitemapEditor(); editor.DeletePage(pageid); editor.Save(); pagesTV.DataSource = source; pagesTV.DataBind(); Table.Visible = false; } } }
/// <summary> /// データ取得 /// </summary> /// <param name="wpd"></param> /// <returns></returns> public List <WebPageData> Get(WebPageData wpd) { try { // 取得データ var retData = new List <WebPageData>(); var dbConnection = new SQLiteConnect(); using (dbConnection.conn) using (var transaction = dbConnection.conn.BeginTransaction()) { var cmd = dbConnection.conn.CreateCommand(); cmd.CommandText = "SELECT * FROM pages "; using (var reader = cmd.ExecuteReader()) { while (reader.Read()) { var temp = new WebPageData(); temp.Title = reader["title"].ToString(); temp.URL = reader["url"].ToString(); retData.Add(temp); } } } return(retData); } catch (Exception ex) { throw ex; } }
public IWebPageReader OnClickCallback(WebPageData item) { return(null); }
public WebPageData[] ExtractData(HtmlDocument doc) { arrangeCurrentState(); List <WebPageData> data = new List <WebPageData>(); switch (currentState) { case CurrentStateInfo.Unknow: break; case CurrentStateInfo.ListOfPerson: AlbumImages.Clear(); var persons = architecture.ListOfPersonReader.GetListOfPerson(doc); foreach (var person in persons) { WebPageData singleData = new WebPageData(); singleData.IsFinal = true; singleData.ImageUrl = person.ImageUrl; singleData.mainText = person.Name; singleData.subText = (person.Description == string.Empty ? "" : person.Description); singleData.NoOfItemsIncluded = person.NoOfAlbums; singleData.UID = MyGlobal.UidGenerator(); singleData.Tag = person; data.Add(singleData); } break; case CurrentStateInfo.ListOfAlbum: if (architecture.ListOfAlbumReader == null) { currentState = CurrentStateInfo.ListOfPerson; return(ExtractData(doc)); } AlbumImages.Clear(); var albums = architecture.ListOfAlbumReader.GetAlbumList((Person)informationForLevel, doc); foreach (var album in albums) { WebPageData singleData = new WebPageData(); singleData.IsFinal = true; singleData.ImageUrl = album.ImageUrl; singleData.mainText = album.Name; singleData.subText = (album.Description == string.Empty ? "" : album.Description); singleData.NoOfItemsIncluded = album.NoOfImages; singleData.UID = MyGlobal.UidGenerator(); singleData.Tag = album; data.Add(singleData); } break; case CurrentStateInfo.ListOfImage: var images = architecture.ListOfImagesReader.GetImageList((Album)informationForLevel, doc); foreach (var img in images) { WebPageData singleData = new WebPageData(); singleData.IsFinal = true; singleData.ImageUrl = img.Image.thumbnil; singleData.mainText = img.Name; singleData.UID = MyGlobal.UidGenerator(); data.Add(singleData); AlbumImages.Add(img.Image); //get the data for the next level and cached it } break; case CurrentStateInfo.ShowImage: break; default: break; } return(data.Count == 0 ? null : data.ToArray()); }
public ProcessBranch(WebPageData wpd) { this.wpd = wpd; }
protected void LoadPageContent() { if (!(File.Exists(HttpContext.Current.Server.MapPath("~/Web.sitemap")))) { SitemapEditor editor = new SitemapEditor(); editor.Save(); } string temp = Context.Items["VirtualPage"].ToString(); if (temp == "/") { temp = "~/default.aspx"; } string pageid = WebPageData.GetWebPageId(temp); _page = WebPageData.LoadPageData(pageid); if (_page.Title.Trim() != string.Empty) { Title = _page.Title; } else { Title = website.WebSiteTitle; } // Load all modules modulelist = ModuleData.GetAllModules(pageid); foreach (int moduleid in modulelist) { Module module = ModuleData.LoadModuleData(moduleid); string filename = ModuleData.GetModuleType(module.ModuleDefinitionId); ModuleControlBaseClass control = new ModuleControlBaseClass(module.ModuleId); control = (ModuleControlBaseClass)LoadControl(filename); control.ModuleId = module.ModuleId; if ((User.IsInRole(module.EditRoles)) || User.IsInRole("administrators")) { control.AdminView = true; } else { control.AdminView = false; } if (!IsPostBack) { control.ViewMode = ViewMode.ReadOnly; } if ((User.IsInRole(module.EditRoles)) || User.IsInRole("administrators")) { CenterArea.Controls.Add(control); } else { if (module.PanelName == "CenterArea") { CenterArea.Controls.Add(control); } if (module.PanelName == "LeftArea") { LeftArea.Controls.Add(control); } if (module.PanelName == "RightArea") { RightArea.Controls.Add(control); } } } HtmlMeta keywords = new HtmlMeta(); keywords.Attributes.Add("name", "keywords"); keywords.Attributes.Add("content", website.Keywords); Header.Controls.Add(keywords); HtmlMeta description = new HtmlMeta(); description.Attributes.Add("name", "description"); description.Attributes.Add("content", website.Description); Header.Controls.Add(description); }