public virtual string ResolveUrl(Pagination pagination = null) { if (pagination == null) { return(this.GetUrl()); } IHtmlAnchorElement nextPageAnchor = null; if (pagination.HasNextPage) { nextPageAnchor = pagination.NextPage as IHtmlAnchorElement; return(nextPageAnchor?.Href); } var currentPage = this.ResolveCurrentPage(pagination.CurrentPage); if (pagination.HasPage(currentPage)) { nextPageAnchor = pagination.Pages[currentPage] as IHtmlAnchorElement; return(nextPageAnchor?.Href); } return(null); }
private static string GetClubIdFrom(IHtmlAnchorElement clubPageUrlEntry) { var uriBuilder = new UriBuilder(clubPageUrlEntry.Href); var queryParameters = HttpUtility.ParseQueryString(uriBuilder.Query); return(queryParameters[ClubIdQueryParameterName]); }
private string GetPlayerIdFrom(IHtmlAnchorElement anchor) { var uriBuilder = new UriBuilder(anchor.Href); var queryParameters = HttpUtility.ParseQueryString(uriBuilder.Query); return(queryParameters[PlayerIdQueryParameterName]); }
public static async Task <ConfirmEmail> Create(IHtmlAnchorElement link, HttpClient client, DefaultUIContext context) { var response = await client.GetAsync(link.Href); var confirmEmail = await ResponseAssert.IsHtmlDocumentAsync(response); return(new ConfirmEmail(client, confirmEmail, context)); }
public Index(HttpClient client, IHtmlDocument manage, HtmlPageContext context) : base(client, manage, context) { _profileLink = HtmlAssert.HasLink("#profile", manage); _changePasswordLink = HtmlAssert.HasLink("#change-password", manage); _twoFactorLink = HtmlAssert.HasLink("#two-factor", manage); _personalDataLink = HtmlAssert.HasLink("#personal-data", manage); }
internal static async Task <ResetPassword> CreateAsync(IHtmlAnchorElement link, HttpClient client, DefaultUIContext context) { var resetPasswordResponse = await client.GetAsync(link.Href); var resetPassword = await ResponseAssert.IsHtmlDocumentAsync(resetPasswordResponse); return(new ResetPassword(client, resetPassword, context)); }
public TwoFactorAuthentication(HttpClient client, IHtmlDocument twoFactor, DefaultUIContext context) : base(client, twoFactor, context) { if (!Context.TwoFactorEnabled) { _enableAuthenticatorLink = HtmlAssert.HasLink("#enable-authenticator", twoFactor); } }
/// <summary> /// Загрузка всей структуры групп товаров /// </summary> /// <returns>Список групп товаров</returns> public static async Task <GroupsCollection> GetGroupsCollection() { GroupsCollection groupsCollection = new GroupsCollection(); if (await groupsCollection.Loader.LoadPageAsync(groupsCollection.Url)) { // Для разбора наименования группы string pattern = @"^(?<name>[^<]+)"; Regex regex = new Regex(pattern); Match match; string categoryName; var catalogBlockList = groupsCollection.Loader.Document.QuerySelectorAll("div") .Where(item => item.ClassName != null && item.ClassName.Contains("category")); foreach (IHtmlDivElement catalogBlock in catalogBlockList) { if (!SiteWorker.CURRENT_INSTANCE.IsActive) { return(groupsCollection); } IHtmlAnchorElement catalogAnchor = (IHtmlAnchorElement)catalogBlock.QuerySelectorAll("a").FirstOrDefault(); if (catalogAnchor != null) { IHtmlParagraphElement catalogLabel = (IHtmlParagraphElement)catalogAnchor.QuerySelectorAll("p").FirstOrDefault(); if (catalogLabel != null) { match = regex.Match(catalogLabel.InnerHtml); if (match.Success) { categoryName = match.Groups["name"].Value.Trim(); Group groupItem = new Group(categoryName, Site.PrepareUrl(catalogAnchor.Href.Trim())); groupsCollection.Groups.Add(groupItem); // Добавление корневой группы groupItem.RegisterGroup(); // Загрузка дочерних групп GroupsCollection subgroupsCollection = await groupItem.GetSubgroupsCollection(); groupsCollection.Groups.AddRange(subgroupsCollection); } } } } // Обновление уровней подчинённости в таблице групп groupsCollection.UpdateDBParents(); } else { string logMessage = @"Ошибка при загрузке страницы: {0}"; SiteWorker.CURRENT_INSTANCE.Log = String.Format(logMessage, groupsCollection.Url); SiteWorker.CURRENT_INSTANCE.Stop(); } return(groupsCollection); }
private Uri ParseUrl(NewsItem currentNew, IHtmlAnchorElement titleAnchor) { if (newsFieldsParser.TryParseUrl(titleAnchor.Href, out Uri url)) { return(url); } return(null); }
public TwoFactorAuthentication(HttpClient client, IHtmlDocument twoFactor, HtmlPageContext context, bool twoFactorEnabled) : base(client, twoFactor, context) { _twoFactorEnabled = twoFactorEnabled; if (!_twoFactorEnabled) { _enableAuthenticatorLink = HtmlAssert.HasLink("#enable-authenticator", twoFactor); } }
private static string GetPlayersPageUrlFrom(IHtmlAnchorElement clubPageUrlEntry) { var uriBuilder = new UriBuilder(clubPageUrlEntry.Href); var queryParameters = HttpUtility.ParseQueryString(uriBuilder.Query); queryParameters[PlayersPageQueryIdParameterName] = PlayersPageQueryIdParameterValue; uriBuilder.Query = queryParameters.ToString(); return(uriBuilder.Uri.ToString()); }
private DateTime ParseDataApresentacao(IHtmlAnchorElement a) { // writetxt(MontaHint('30/09/2003','Legislação Societária','Apresentação ','29/10/2003 13:07','P')); var javascript = a.GetAttribute("onmouseover"); var items = javascript.Split(','); var date = items[3].Replace("\"", "").Replace("\'", ""); var data = DateTime.ParseExact(date, "dd/MM/yyyy H:mm", new CultureInfo("pt-BR")); return(data); }
public DocLinkInfo(DocInfoType doctype, IHtmlAnchorElement a) { Title = a.Text; DocType = doctype; Data = ParseDocLinkDate(a.Text); DataApresentacao = ParseDataApresentacao(a); Link = ParseDocLinkHref(a.Href); LinkType = ParseLinkType(Link); if (LinkType == LinkTypeEnum.CVM) { NumeroSequencialDocumento = ParseNumSequencial(Link); } }
public async Task ContainsLinks(Page page) { var response = await RetryHelper.RetryRequest(async() => { var request = new HttpRequestMessage( HttpMethod.Get, new Uri(ListeningUri, page.Url)); return(await _httpClient.SendAsync(request)); }, logger : NullLogger.Instance); Assert.Equal(HttpStatusCode.OK, response.StatusCode); var parser = new HtmlParser(); var html = await parser.ParseAsync(await response.Content.ReadAsStreamAsync()); foreach (IHtmlLinkElement styleSheet in html.GetElementsByTagName("link")) { Assert.Equal("stylesheet", styleSheet.Relation); // Workaround for https://github.com/dotnet/aspnetcore/issues/31030#issuecomment-811334450 // Cleans up incorrectly generated filename for scoped CSS files var styleSheetHref = styleSheet.Href.Replace("_", string.Empty).Replace("about://", string.Empty); await AssertOk(styleSheetHref); } foreach (var script in html.Scripts) { if (!string.IsNullOrEmpty(script.Source)) { await AssertOk(script.Source); } } Assert.True(html.Links.Length == page.Links.Count(), $"Expected {page.Url} to have {page.Links.Count()} links but it had {html.Links.Length}"); foreach ((var link, var expectedLink) in html.Links.Zip(page.Links, Tuple.Create)) { IHtmlAnchorElement anchor = (IHtmlAnchorElement)link; if (string.Equals(anchor.Protocol, "about:")) { Assert.True(anchor.PathName.EndsWith(expectedLink, StringComparison.Ordinal), $"Expected next link on {page.Url} to be {expectedLink} but it was {anchor.PathName}: {html.Source.Text}"); await AssertOk(anchor.PathName); } else { Assert.True(string.Equals(anchor.Href, expectedLink), $"Expected next link to be {expectedLink} but it was {anchor.Href}."); var result = await RetryHelper.RetryRequest(async() => { return(await _httpClient.GetAsync(anchor.Href)); }, logger : NullLogger.Instance); Assert.True(IsSuccessStatusCode(result), $"{anchor.Href} is a broken link!"); } } }
public Login( HttpClient client, IHtmlDocument login, DefaultUIContext context) : base(client, login, context) { _loginForm = HtmlAssert.HasForm("#account", login); _forgotPasswordLink = HtmlAssert.HasLink("#forgot-password", login); if (Context.ContosoLoginEnabled) { _externalLoginForm = HtmlAssert.HasForm("#external-account", login); _contosoButton = HtmlAssert.HasElement("button[value=Contoso]", login); } }
public Index(HttpClient client, IHtmlDocument index, HtmlPageContext context, bool authenticated) : base(client, index, context) { _authenticated = authenticated; if (!_authenticated) { _registerLink = HtmlAssert.HasLink("#register", Document); _loginLink = HtmlAssert.HasLink("#login", Document); } else { _manageLink = HtmlAssert.HasLink("#manage", Document); } }
public Index(HttpClient client, IHtmlDocument manage, DefaultUIContext context) : base(client, manage, context) { Assert.True(Context.UserAuthenticated); _profileLink = HtmlAssert.HasLink("#profile", manage); _changePasswordLink = HtmlAssert.HasLink("#change-password", manage); _twoFactorLink = HtmlAssert.HasLink("#two-factor", manage); _personalDataLink = HtmlAssert.HasLink("#personal-data", manage); _updateProfileForm = HtmlAssert.HasForm("#profile-form", manage); if (!Context.EmailConfirmed) { _confirmEmailButton = HtmlAssert.HasElement("button#email-verification", manage); } }
public RegisterConfirmation( HttpClient client, IHtmlDocument register, DefaultUIContext context) : base(client, register, context) { if (Context.HasRealEmailSender) { Assert.Empty(Document.QuerySelectorAll("#confirm-link")); } else { _confirmLink = HtmlAssert.HasLink("#confirm-link", Document); } }
public async Task ContainsLinks(Page page) { var response = await RequestWithRetries(client => { var request = new HttpRequestMessage( HttpMethod.Get, new Uri(ListeningUri, page.Url)); return(client.SendAsync(request)); }, _httpClient); Assert.Equal(HttpStatusCode.OK, response.StatusCode); var parser = new HtmlParser(); var html = await parser.ParseAsync(await response.Content.ReadAsStreamAsync()); foreach (IHtmlLinkElement styleSheet in html.GetElementsByTagName("link")) { Assert.Equal("stylesheet", styleSheet.Relation); await AssertOk(styleSheet.Href.Replace("about://", string.Empty)); } foreach (var script in html.Scripts) { if (!string.IsNullOrEmpty(script.Source)) { await AssertOk(script.Source); } } Assert.True(html.Links.Length == page.Links.Count(), $"Expected {page.Url} to have {page.Links.Count()} links but it had {html.Links.Length}"); foreach ((var link, var expectedLink) in html.Links.Zip(page.Links, Tuple.Create)) { IHtmlAnchorElement anchor = (IHtmlAnchorElement)link; if (string.Equals(anchor.Protocol, "about:")) { Assert.True(anchor.PathName.EndsWith(expectedLink), $"Expected next link on {page.Url} to be {expectedLink} but it was {anchor.PathName}."); await AssertOk(anchor.PathName); } else { Assert.True(string.Equals(anchor.Href, expectedLink), $"Expected next link to be {expectedLink} but it was {anchor.Href}."); var result = await RetryHelper.RetryRequest(async() => { return(await RequestWithRetries(client => client.GetAsync(anchor.Href), _httpClient)); }, logger : NullLogger.Instance); Assert.True(IsSuccessStatusCode(result), $"{anchor.Href} is a broken link!"); } } }
public Index( HttpClient client, IHtmlDocument index, DefaultUIContext context) : base(client, index, context) { if (!Context.UserAuthenticated) { _registerLink = HtmlAssert.HasLink("#register", Document); _loginLink = HtmlAssert.HasLink("#login", Document); } else { _manageLink = HtmlAssert.HasLink("#manage", Document); } }
private Inline GenerateLink(IHtmlAnchorElement node) { var hyperlink = new Hyperlink(); if (Uri.TryCreate(node.Href, UriKind.RelativeOrAbsolute, out Uri hrefUri)) { hyperlink.NavigateUri = hrefUri; } // TODO: Add option for unfurling links as images // TODO: Add link clicked event AddInlineChildren(node, hyperlink.Inlines); return(hyperlink); }
private static Dictionary <string, TableBox> GetPackages(Dictionary <string, TableBox> Alte) { try { IDocument document = BrowsingContext.New(Configuration.Default.WithDefaultLoader()).OpenAsync(ModelPage).Result; string PreviousName = null; return(document.QuerySelector <IHtmlTableElement>("table.wiki").Rows.Where(x => x.Cells.Length == 10 && !x.Cells[0].TextContent.Contains("Modell")).Select(row => { var cells = row.Cells; TableBox box = new TableBox(); box.Name = cells[0].TextContent.Trim(); //Wenn die Zelle nur + enthält dann ist damit der name der vorherigen Zeile gemeint if (box.Name == "+" && !String.IsNullOrEmpty(PreviousName)) { box.Name = PreviousName; } PreviousName = box.Name; box.FreetzType = cells[1].TextContent.Trim(); IHtmlAnchorElement link = cells[1].ChildNodes.OfType <IHtmlAnchorElement>().FirstOrDefault(); if (link != null) { box.Url = link.HrefOrNull(); } box.AngepassteFirmware = cells[2].TextContent.Trim(); box.FreetzVersion = cells[3].TextContent.Trim(); box.Annex = cells[4].TextContent.Trim(); box.Sprache = cells[5].TextContent.Trim(); box.CPU = cells[6].TextContent.Trim(); box.Flash = cells[7].TextContent.Trim(); box.RAM = cells[8].TextContent.Trim(); box.USBHost = cells[9].TextContent.Trim(); return box; }).Distinct(x => x.FreetzType).ToDictionary(k => k.FreetzType, v => v, StringComparer.OrdinalIgnoreCase)); } catch { return(Alte); } }
public Index(HttpClient client, IHtmlDocument manage, DefaultUIContext context) : base(client, manage, context) { Assert.True(Context.UserAuthenticated); _profileLink = HtmlAssert.HasLink("#profile", manage); _emailLink = HtmlAssert.HasLink("#email", manage); _changePasswordLink = HtmlAssert.HasLink("#change-password", manage); _twoFactorLink = HtmlAssert.HasLink("#two-factor", manage); if (Context.ContosoLoginEnabled) { _externalLoginLink = HtmlAssert.HasLink("#external-login", manage); } _personalDataLink = HtmlAssert.HasLink("#personal-data", manage); _updateProfileForm = HtmlAssert.HasForm("#profile-form", manage); _userNameInput = HtmlAssert.HasElement("#Username", manage); _updateProfileButton = HtmlAssert.HasElement("#update-profile-button", manage); }
public async Task <IDocument> NavigateToNextPage(IDocument document) { // Obter todos os botões de navegação IHtmlCollection <IElement> buttonsToNav = document.QuerySelectorAll("li.pagination-item"); // Obter o último IElement buttonNextPage = buttonsToNav.Last(); // Obter o elemento ancora dentro do botao de navegação IHtmlAnchorElement buttonGoNextPage = buttonNextPage.ChildNodes.OfType <IHtmlAnchorElement>().FirstOrDefault(); // Vamos tentar retirar o número da página e colocá-la numa varíavel try { string nextWebsiteLinkPage = buttonGoNextPage.GetAttribute("href"); string toBeSearched = "pag="; string numberPageString = nextWebsiteLinkPage.Substring(nextWebsiteLinkPage.IndexOf(toBeSearched) + toBeSearched.Length); if (numberPageString != string.Empty) { if (Convert.ToInt32(numberPageString) > NumberPage) { NumberPage = Convert.ToInt32(numberPageString); } // Para evitar um eterno loop. else { return(null); } } } catch (FormatException ex) { _ = new LogWriter("format", ex.Message.ToString()); } // Navegar para a página return(await(buttonGoNextPage?.NavigateAsync())); }
public TwoFactorAuthentication(HttpClient client, IHtmlDocument twoFactor, DefaultUIContext context) : base(client, twoFactor, context) { if (Context.CookiePolicyAccepted) { if (!Context.TwoFactorEnabled) { _enableAuthenticatorLink = HtmlAssert.HasLink("#enable-authenticator", twoFactor); } else { _resetAuthenticatorLink = HtmlAssert.HasLink("#reset-authenticator", twoFactor); } } else { Assert.Contains("You must accept the policy before you can enable two factor authentication.", twoFactor.DocumentElement.TextContent); } }
public async void ParseAsync(string html) { HtmlParser domParser = new HtmlParser(); IHtmlDocument document = await domParser.ParseDocumentAsync(html); IHtmlAnchorElement downEl = document.QuerySelectorAll <IHtmlAnchorElement>("a") .Where(item => item.ClassName != null && item.ClassName.Contains("musicset-track__download-link track-geo__button")).FirstOrDefault(); string res = downEl.PathName; if (downEl == default) { ParseFailed?.Invoke(); } else { ParseSuccess?.Invoke(); DownloadFromSite(res); } }
async Task <IEnumerable <IDeal> > IScraper.Scrape(CancellationToken token) { Url url = Url.Create("https://www.gog.com"); IDocument document = await context.OpenAsync(url, token); token.ThrowIfCancellationRequested(); IHtmlAnchorElement giveawayAnchor = document.Body.QuerySelector <IHtmlAnchorElement>(".giveaway-banner"); if (giveawayAnchor == null) { logger.Info("No giveaway found"); return(new List <IDeal>()); } string onclickContent = giveawayAnchor.GetAttribute("onclick"); Match match = Regex.Match(onclickContent, "'.+'"); if (!match.Success) { logger.Info("No onclick found"); return(new List <IDeal>()); } string encodedJson = match.Value.Trim('\''); string json = Regex.Unescape(encodedJson); GoodOldGamesData data = JsonConvert.DeserializeObject <GoodOldGamesData>(json); return(new List <IDeal> { new Deal { Image = $"https://images-1.gog-statics.com/{data.Logo.Image}.png", Link = $"https://www.gog.com{data.GameUrl}", Title = data.Title, End = DateTimeOffset.FromUnixTimeMilliseconds(data.EndTime).UtcDateTime } }); }
public IList <NewsItem> ParseNews(IDocument document) { var rows = document.QuerySelectorAll("table.itemlist tr"); var news = new List <NewsItem>(); NewsItem currentNew = null; foreach (var row in rows) { if (row.ClassName == "athing") { IHtmlAnchorElement titleAnchor = row.QuerySelector("a.storylink") as IHtmlAnchorElement; currentNew = new NewsItem { Title = newsFieldsParser.ParseTitle(titleAnchor?.TextContent), Rank = ParseRank(currentNew, row), Url = ParseUrl(currentNew, titleAnchor), Id = row.Id }; } if (currentNew != null && row.QuerySelectorAll($"#score_{currentNew.Id}").Any()) { currentNew.Points = ParsePoints(currentNew, row); currentNew.Comments = ParseComments(currentNew, row); currentNew.User = newsFieldsParser.ParseUser(row.QuerySelector($".hnuser").TextContent); if (currentNew.IsValid()) { news.Add(currentNew); } } } return(news); }
static Boolean IsAnchor(IHtmlAnchorElement element) { return element.Attributes.Any(m => String.Equals(m.Name, AttributeNames.Name, StringComparison.Ordinal)); }
static Boolean IsAnchor(IHtmlAnchorElement element) { return(element.Attributes.Any(m => m.Name.Is(AttributeNames.Name))); }
static Boolean IsAnchor(IHtmlAnchorElement element) { return element.Attributes.Any(m => m.Name == AttributeNames.Name); }
public override async Task <MemeInfo> RandomAsync() { WebClient wc; string html; try { wc = new WebClient(); html = await wc.DownloadStringTaskAsync(_uris[UriType.Random]); } catch (WebException ex) { throw new ServiceOrConnectionException("Could not load the page", ex); } IConfiguration config = Configuration.Default; IBrowsingContext context = BrowsingContext.New(config); IDocument document = await context.OpenAsync(req => req.Content(html).Address(_baseUrl)); IElement picDiv = document.DocumentElement .QuerySelectorAll("#wrapper-wrap .left .ob-left-box-images")[RandomNthChild()]; IHtmlImageElement img = (IHtmlImageElement)picDiv .QuerySelector(".left-wrap a img:last-child"); IHtmlAnchorElement a = (IHtmlAnchorElement)picDiv .QuerySelector("h2 a"); IHtmlInputElement input = (IHtmlInputElement)picDiv .QuerySelector(".left-wrap input[type=\"hidden\"]"); IHtmlSourceElement src = (IHtmlSourceElement)picDiv .QuerySelector(".left-wrap video > source"); if ((src == null && img == null) || a == null) { throw new NotFoundException( "Either \"img\", \"source\" or \"a\" tag could not be found"); } MemeInfo meme; if (src != null) { meme = new MemeInfo { ViewURI = a.Href, URI = src.Source, Alt = string.Empty, Name = a.TextContent, Type = MediaType.Video }; } else if (input != null) { meme = new MemeInfo { ViewURI = a.Href, URI = input.Value, Alt = img.AlternativeText, Name = a.TextContent, Type = MediaType.Gif }; } else { meme = new MemeInfo { ViewURI = a.Href, URI = img.Source, Alt = img.AlternativeText, Name = a.TextContent, Type = MediaType.Image }; } return(meme); }