public virtual string ResolveUrl(Pagination pagination = null)
{
if (pagination == null)
{
return(this.GetUrl());
}
IHtmlAnchorElement nextPageAnchor = null;
if (pagination.HasNextPage)
{
nextPageAnchor = pagination.NextPage as IHtmlAnchorElement;
return(nextPageAnchor?.Href);
}
var currentPage = this.ResolveCurrentPage(pagination.CurrentPage);
if (pagination.HasPage(currentPage))
{
nextPageAnchor = pagination.Pages[currentPage] as IHtmlAnchorElement;
return(nextPageAnchor?.Href);
}
return(null);
}
private static string GetClubIdFrom(IHtmlAnchorElement clubPageUrlEntry)
{
var uriBuilder = new UriBuilder(clubPageUrlEntry.Href);
var queryParameters = HttpUtility.ParseQueryString(uriBuilder.Query);
return(queryParameters[ClubIdQueryParameterName]);
}
private string GetPlayerIdFrom(IHtmlAnchorElement anchor)
{
var uriBuilder = new UriBuilder(anchor.Href);
var queryParameters = HttpUtility.ParseQueryString(uriBuilder.Query);
return(queryParameters[PlayerIdQueryParameterName]);
}
public static async Task <ConfirmEmail> Create(IHtmlAnchorElement link, HttpClient client, DefaultUIContext context)
{
var response = await client.GetAsync(link.Href);
var confirmEmail = await ResponseAssert.IsHtmlDocumentAsync(response);
return(new ConfirmEmail(client, confirmEmail, context));
}
public Index(HttpClient client, IHtmlDocument manage, HtmlPageContext context)
: base(client, manage, context)
{
_profileLink = HtmlAssert.HasLink("#profile", manage);
_changePasswordLink = HtmlAssert.HasLink("#change-password", manage);
_twoFactorLink = HtmlAssert.HasLink("#two-factor", manage);
_personalDataLink = HtmlAssert.HasLink("#personal-data", manage);
}
internal static async Task <ResetPassword> CreateAsync(IHtmlAnchorElement link, HttpClient client, DefaultUIContext context)
{
var resetPasswordResponse = await client.GetAsync(link.Href);
var resetPassword = await ResponseAssert.IsHtmlDocumentAsync(resetPasswordResponse);
return(new ResetPassword(client, resetPassword, context));
}
public TwoFactorAuthentication(HttpClient client, IHtmlDocument twoFactor, DefaultUIContext context)
: base(client, twoFactor, context)
{
if (!Context.TwoFactorEnabled)
{
_enableAuthenticatorLink = HtmlAssert.HasLink("#enable-authenticator", twoFactor);
}
}
/// <summary>
/// Загрузка всей структуры групп товаров
/// </summary>
/// <returns>Список групп товаров</returns>
public static async Task <GroupsCollection> GetGroupsCollection()
{
GroupsCollection groupsCollection = new GroupsCollection();
if (await groupsCollection.Loader.LoadPageAsync(groupsCollection.Url))
{
// Для разбора наименования группы
string pattern = @"^(?<name>[^<]+)";
Regex regex = new Regex(pattern);
Match match;
string categoryName;
var catalogBlockList = groupsCollection.Loader.Document.QuerySelectorAll("div")
.Where(item => item.ClassName != null &&
item.ClassName.Contains("category"));
foreach (IHtmlDivElement catalogBlock in catalogBlockList)
{
if (!SiteWorker.CURRENT_INSTANCE.IsActive)
{
return(groupsCollection);
}
IHtmlAnchorElement catalogAnchor = (IHtmlAnchorElement)catalogBlock.QuerySelectorAll("a").FirstOrDefault();
if (catalogAnchor != null)
{
IHtmlParagraphElement catalogLabel = (IHtmlParagraphElement)catalogAnchor.QuerySelectorAll("p").FirstOrDefault();
if (catalogLabel != null)
{
match = regex.Match(catalogLabel.InnerHtml);
if (match.Success)
{
categoryName = match.Groups["name"].Value.Trim();
Group groupItem = new Group(categoryName, Site.PrepareUrl(catalogAnchor.Href.Trim()));
groupsCollection.Groups.Add(groupItem); // Добавление корневой группы
groupItem.RegisterGroup();
// Загрузка дочерних групп
GroupsCollection subgroupsCollection = await groupItem.GetSubgroupsCollection();
groupsCollection.Groups.AddRange(subgroupsCollection);
}
}
}
}
// Обновление уровней подчинённости в таблице групп
groupsCollection.UpdateDBParents();
}
else
{
string logMessage = @"Ошибка при загрузке страницы: {0}";
SiteWorker.CURRENT_INSTANCE.Log = String.Format(logMessage, groupsCollection.Url);
SiteWorker.CURRENT_INSTANCE.Stop();
}
return(groupsCollection);
}
private Uri ParseUrl(NewsItem currentNew, IHtmlAnchorElement titleAnchor)
{
if (newsFieldsParser.TryParseUrl(titleAnchor.Href, out Uri url))
{
return(url);
}
return(null);
}
public TwoFactorAuthentication(HttpClient client, IHtmlDocument twoFactor, HtmlPageContext context, bool twoFactorEnabled)
: base(client, twoFactor, context)
{
_twoFactorEnabled = twoFactorEnabled;
if (!_twoFactorEnabled)
{
_enableAuthenticatorLink = HtmlAssert.HasLink("#enable-authenticator", twoFactor);
}
}
private static string GetPlayersPageUrlFrom(IHtmlAnchorElement clubPageUrlEntry)
{
var uriBuilder = new UriBuilder(clubPageUrlEntry.Href);
var queryParameters = HttpUtility.ParseQueryString(uriBuilder.Query);
queryParameters[PlayersPageQueryIdParameterName] = PlayersPageQueryIdParameterValue;
uriBuilder.Query = queryParameters.ToString();
return(uriBuilder.Uri.ToString());
}
private DateTime ParseDataApresentacao(IHtmlAnchorElement a)
{
// writetxt(MontaHint('30/09/2003','Legislação Societária','Apresentação ','29/10/2003 13:07','P'));
var javascript = a.GetAttribute("onmouseover");
var items = javascript.Split(',');
var date = items[3].Replace("\"", "").Replace("\'", "");
var data = DateTime.ParseExact(date, "dd/MM/yyyy H:mm", new CultureInfo("pt-BR"));
return(data);
}
public DocLinkInfo(DocInfoType doctype, IHtmlAnchorElement a)
{
Title = a.Text;
DocType = doctype;
Data = ParseDocLinkDate(a.Text);
DataApresentacao = ParseDataApresentacao(a);
Link = ParseDocLinkHref(a.Href);
LinkType = ParseLinkType(Link);
if (LinkType == LinkTypeEnum.CVM)
{
NumeroSequencialDocumento = ParseNumSequencial(Link);
}
}
public async Task ContainsLinks(Page page)
{
var response = await RetryHelper.RetryRequest(async() =>
{
var request = new HttpRequestMessage(
HttpMethod.Get,
new Uri(ListeningUri, page.Url));
return(await _httpClient.SendAsync(request));
}, logger : NullLogger.Instance);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var parser = new HtmlParser();
var html = await parser.ParseAsync(await response.Content.ReadAsStreamAsync());
foreach (IHtmlLinkElement styleSheet in html.GetElementsByTagName("link"))
{
Assert.Equal("stylesheet", styleSheet.Relation);
// Workaround for https://github.com/dotnet/aspnetcore/issues/31030#issuecomment-811334450
// Cleans up incorrectly generated filename for scoped CSS files
var styleSheetHref = styleSheet.Href.Replace("_", string.Empty).Replace("about://", string.Empty);
await AssertOk(styleSheetHref);
}
foreach (var script in html.Scripts)
{
if (!string.IsNullOrEmpty(script.Source))
{
await AssertOk(script.Source);
}
}
Assert.True(html.Links.Length == page.Links.Count(), $"Expected {page.Url} to have {page.Links.Count()} links but it had {html.Links.Length}");
foreach ((var link, var expectedLink) in html.Links.Zip(page.Links, Tuple.Create))
{
IHtmlAnchorElement anchor = (IHtmlAnchorElement)link;
if (string.Equals(anchor.Protocol, "about:"))
{
Assert.True(anchor.PathName.EndsWith(expectedLink, StringComparison.Ordinal), $"Expected next link on {page.Url} to be {expectedLink} but it was {anchor.PathName}: {html.Source.Text}");
await AssertOk(anchor.PathName);
}
else
{
Assert.True(string.Equals(anchor.Href, expectedLink), $"Expected next link to be {expectedLink} but it was {anchor.Href}.");
var result = await RetryHelper.RetryRequest(async() =>
{
return(await _httpClient.GetAsync(anchor.Href));
}, logger : NullLogger.Instance);
Assert.True(IsSuccessStatusCode(result), $"{anchor.Href} is a broken link!");
}
}
}
public Login(
HttpClient client,
IHtmlDocument login,
DefaultUIContext context)
: base(client, login, context)
{
_loginForm = HtmlAssert.HasForm("#account", login);
_forgotPasswordLink = HtmlAssert.HasLink("#forgot-password", login);
if (Context.ContosoLoginEnabled)
{
_externalLoginForm = HtmlAssert.HasForm("#external-account", login);
_contosoButton = HtmlAssert.HasElement("button[value=Contoso]", login);
}
}
public Index(HttpClient client, IHtmlDocument index, HtmlPageContext context, bool authenticated)
: base(client, index, context)
{
_authenticated = authenticated;
if (!_authenticated)
{
_registerLink = HtmlAssert.HasLink("#register", Document);
_loginLink = HtmlAssert.HasLink("#login", Document);
}
else
{
_manageLink = HtmlAssert.HasLink("#manage", Document);
}
}
public Index(HttpClient client, IHtmlDocument manage, DefaultUIContext context)
: base(client, manage, context)
{
Assert.True(Context.UserAuthenticated);
_profileLink = HtmlAssert.HasLink("#profile", manage);
_changePasswordLink = HtmlAssert.HasLink("#change-password", manage);
_twoFactorLink = HtmlAssert.HasLink("#two-factor", manage);
_personalDataLink = HtmlAssert.HasLink("#personal-data", manage);
_updateProfileForm = HtmlAssert.HasForm("#profile-form", manage);
if (!Context.EmailConfirmed)
{
_confirmEmailButton = HtmlAssert.HasElement("button#email-verification", manage);
}
}
public RegisterConfirmation(
HttpClient client,
IHtmlDocument register,
DefaultUIContext context)
: base(client, register, context)
{
if (Context.HasRealEmailSender)
{
Assert.Empty(Document.QuerySelectorAll("#confirm-link"));
}
else
{
_confirmLink = HtmlAssert.HasLink("#confirm-link", Document);
}
}
public async Task ContainsLinks(Page page)
{
var response = await RequestWithRetries(client =>
{
var request = new HttpRequestMessage(
HttpMethod.Get,
new Uri(ListeningUri, page.Url));
return(client.SendAsync(request));
}, _httpClient);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var parser = new HtmlParser();
var html = await parser.ParseAsync(await response.Content.ReadAsStreamAsync());
foreach (IHtmlLinkElement styleSheet in html.GetElementsByTagName("link"))
{
Assert.Equal("stylesheet", styleSheet.Relation);
await AssertOk(styleSheet.Href.Replace("about://", string.Empty));
}
foreach (var script in html.Scripts)
{
if (!string.IsNullOrEmpty(script.Source))
{
await AssertOk(script.Source);
}
}
Assert.True(html.Links.Length == page.Links.Count(), $"Expected {page.Url} to have {page.Links.Count()} links but it had {html.Links.Length}");
foreach ((var link, var expectedLink) in html.Links.Zip(page.Links, Tuple.Create))
{
IHtmlAnchorElement anchor = (IHtmlAnchorElement)link;
if (string.Equals(anchor.Protocol, "about:"))
{
Assert.True(anchor.PathName.EndsWith(expectedLink), $"Expected next link on {page.Url} to be {expectedLink} but it was {anchor.PathName}.");
await AssertOk(anchor.PathName);
}
else
{
Assert.True(string.Equals(anchor.Href, expectedLink), $"Expected next link to be {expectedLink} but it was {anchor.Href}.");
var result = await RetryHelper.RetryRequest(async() =>
{
return(await RequestWithRetries(client => client.GetAsync(anchor.Href), _httpClient));
}, logger : NullLogger.Instance);
Assert.True(IsSuccessStatusCode(result), $"{anchor.Href} is a broken link!");
}
}
}
public Index(
HttpClient client,
IHtmlDocument index,
DefaultUIContext context)
: base(client, index, context)
{
if (!Context.UserAuthenticated)
{
_registerLink = HtmlAssert.HasLink("#register", Document);
_loginLink = HtmlAssert.HasLink("#login", Document);
}
else
{
_manageLink = HtmlAssert.HasLink("#manage", Document);
}
}
private Inline GenerateLink(IHtmlAnchorElement node)
{
var hyperlink = new Hyperlink();
if (Uri.TryCreate(node.Href, UriKind.RelativeOrAbsolute, out Uri hrefUri))
{
hyperlink.NavigateUri = hrefUri;
}
// TODO: Add option for unfurling links as images
// TODO: Add link clicked event
AddInlineChildren(node, hyperlink.Inlines);
return(hyperlink);
}
private static Dictionary <string, TableBox> GetPackages(Dictionary <string, TableBox> Alte)
{
try
{
IDocument document = BrowsingContext.New(Configuration.Default.WithDefaultLoader()).OpenAsync(ModelPage).Result;
string PreviousName = null;
return(document.QuerySelector <IHtmlTableElement>("table.wiki").Rows.Where(x => x.Cells.Length == 10 && !x.Cells[0].TextContent.Contains("Modell")).Select(row =>
{
var cells = row.Cells;
TableBox box = new TableBox();
box.Name = cells[0].TextContent.Trim();
//Wenn die Zelle nur + enthält dann ist damit der name der vorherigen Zeile gemeint
if (box.Name == "+" && !String.IsNullOrEmpty(PreviousName))
{
box.Name = PreviousName;
}
PreviousName = box.Name;
box.FreetzType = cells[1].TextContent.Trim();
IHtmlAnchorElement link = cells[1].ChildNodes.OfType <IHtmlAnchorElement>().FirstOrDefault();
if (link != null)
{
box.Url = link.HrefOrNull();
}
box.AngepassteFirmware = cells[2].TextContent.Trim();
box.FreetzVersion = cells[3].TextContent.Trim();
box.Annex = cells[4].TextContent.Trim();
box.Sprache = cells[5].TextContent.Trim();
box.CPU = cells[6].TextContent.Trim();
box.Flash = cells[7].TextContent.Trim();
box.RAM = cells[8].TextContent.Trim();
box.USBHost = cells[9].TextContent.Trim();
return box;
}).Distinct(x => x.FreetzType).ToDictionary(k => k.FreetzType, v => v, StringComparer.OrdinalIgnoreCase));
}
catch
{
return(Alte);
}
}
public Index(HttpClient client, IHtmlDocument manage, DefaultUIContext context)
: base(client, manage, context)
{
Assert.True(Context.UserAuthenticated);
_profileLink = HtmlAssert.HasLink("#profile", manage);
_emailLink = HtmlAssert.HasLink("#email", manage);
_changePasswordLink = HtmlAssert.HasLink("#change-password", manage);
_twoFactorLink = HtmlAssert.HasLink("#two-factor", manage);
if (Context.ContosoLoginEnabled)
{
_externalLoginLink = HtmlAssert.HasLink("#external-login", manage);
}
_personalDataLink = HtmlAssert.HasLink("#personal-data", manage);
_updateProfileForm = HtmlAssert.HasForm("#profile-form", manage);
_userNameInput = HtmlAssert.HasElement("#Username", manage);
_updateProfileButton = HtmlAssert.HasElement("#update-profile-button", manage);
}
public async Task <IDocument> NavigateToNextPage(IDocument document)
{
// Obter todos os botões de navegação
IHtmlCollection <IElement> buttonsToNav = document.QuerySelectorAll("li.pagination-item");
// Obter o último
IElement buttonNextPage = buttonsToNav.Last();
// Obter o elemento ancora dentro do botao de navegação
IHtmlAnchorElement buttonGoNextPage = buttonNextPage.ChildNodes.OfType <IHtmlAnchorElement>().FirstOrDefault();
// Vamos tentar retirar o número da página e colocá-la numa varíavel
try
{
string nextWebsiteLinkPage = buttonGoNextPage.GetAttribute("href");
string toBeSearched = "pag=";
string numberPageString = nextWebsiteLinkPage.Substring(nextWebsiteLinkPage.IndexOf(toBeSearched) + toBeSearched.Length);
if (numberPageString != string.Empty)
{
if (Convert.ToInt32(numberPageString) > NumberPage)
{
NumberPage = Convert.ToInt32(numberPageString);
}
// Para evitar um eterno loop.
else
{
return(null);
}
}
}
catch (FormatException ex)
{
_ = new LogWriter("format", ex.Message.ToString());
}
// Navegar para a página
return(await(buttonGoNextPage?.NavigateAsync()));
}
public TwoFactorAuthentication(HttpClient client, IHtmlDocument twoFactor, DefaultUIContext context)
: base(client, twoFactor, context)
{
if (Context.CookiePolicyAccepted)
{
if (!Context.TwoFactorEnabled)
{
_enableAuthenticatorLink = HtmlAssert.HasLink("#enable-authenticator", twoFactor);
}
else
{
_resetAuthenticatorLink = HtmlAssert.HasLink("#reset-authenticator", twoFactor);
}
}
else
{
Assert.Contains("You must accept the policy before you can enable two factor authentication.", twoFactor.DocumentElement.TextContent);
}
}
public async void ParseAsync(string html)
{
HtmlParser domParser = new HtmlParser();
IHtmlDocument document = await domParser.ParseDocumentAsync(html);
IHtmlAnchorElement downEl = document.QuerySelectorAll <IHtmlAnchorElement>("a")
.Where(item => item.ClassName != null && item.ClassName.Contains("musicset-track__download-link track-geo__button")).FirstOrDefault();
string res = downEl.PathName;
if (downEl == default)
{
ParseFailed?.Invoke();
}
else
{
ParseSuccess?.Invoke();
DownloadFromSite(res);
}
}
async Task <IEnumerable <IDeal> > IScraper.Scrape(CancellationToken token)
{
Url url = Url.Create("https://www.gog.com");
IDocument document = await context.OpenAsync(url, token);
token.ThrowIfCancellationRequested();
IHtmlAnchorElement giveawayAnchor = document.Body.QuerySelector <IHtmlAnchorElement>(".giveaway-banner");
if (giveawayAnchor == null)
{
logger.Info("No giveaway found");
return(new List <IDeal>());
}
string onclickContent = giveawayAnchor.GetAttribute("onclick");
Match match = Regex.Match(onclickContent, "'.+'");
if (!match.Success)
{
logger.Info("No onclick found");
return(new List <IDeal>());
}
string encodedJson = match.Value.Trim('\'');
string json = Regex.Unescape(encodedJson);
GoodOldGamesData data = JsonConvert.DeserializeObject <GoodOldGamesData>(json);
return(new List <IDeal>
{
new Deal
{
Image = $"https://images-1.gog-statics.com/{data.Logo.Image}.png",
Link = $"https://www.gog.com{data.GameUrl}",
Title = data.Title,
End = DateTimeOffset.FromUnixTimeMilliseconds(data.EndTime).UtcDateTime
}
});
}
public IList <NewsItem> ParseNews(IDocument document)
{
var rows = document.QuerySelectorAll("table.itemlist tr");
var news = new List <NewsItem>();
NewsItem currentNew = null;
foreach (var row in rows)
{
if (row.ClassName == "athing")
{
IHtmlAnchorElement titleAnchor = row.QuerySelector("a.storylink") as IHtmlAnchorElement;
currentNew = new NewsItem
{
Title = newsFieldsParser.ParseTitle(titleAnchor?.TextContent),
Rank = ParseRank(currentNew, row),
Url = ParseUrl(currentNew, titleAnchor),
Id = row.Id
};
}
if (currentNew != null && row.QuerySelectorAll($"#score_{currentNew.Id}").Any())
{
currentNew.Points = ParsePoints(currentNew, row);
currentNew.Comments = ParseComments(currentNew, row);
currentNew.User = newsFieldsParser.ParseUser(row.QuerySelector($".hnuser").TextContent);
if (currentNew.IsValid())
{
news.Add(currentNew);
}
}
}
return(news);
}
static Boolean IsAnchor(IHtmlAnchorElement element)
{
return element.Attributes.Any(m => String.Equals(m.Name, AttributeNames.Name, StringComparison.Ordinal));
}
static Boolean IsAnchor(IHtmlAnchorElement element)
{
return(element.Attributes.Any(m => m.Name.Is(AttributeNames.Name)));
}
static Boolean IsAnchor(IHtmlAnchorElement element)
{
return element.Attributes.Any(m => m.Name == AttributeNames.Name);
}
public override async Task <MemeInfo> RandomAsync()
{
WebClient wc;
string html;
try
{
wc = new WebClient();
html = await wc.DownloadStringTaskAsync(_uris[UriType.Random]);
}
catch (WebException ex)
{
throw new ServiceOrConnectionException("Could not load the page", ex);
}
IConfiguration config = Configuration.Default;
IBrowsingContext context = BrowsingContext.New(config);
IDocument document = await context.OpenAsync(req => req.Content(html).Address(_baseUrl));
IElement picDiv = document.DocumentElement
.QuerySelectorAll("#wrapper-wrap .left .ob-left-box-images")[RandomNthChild()];
IHtmlImageElement img = (IHtmlImageElement)picDiv
.QuerySelector(".left-wrap a img:last-child");
IHtmlAnchorElement a = (IHtmlAnchorElement)picDiv
.QuerySelector("h2 a");
IHtmlInputElement input = (IHtmlInputElement)picDiv
.QuerySelector(".left-wrap input[type=\"hidden\"]");
IHtmlSourceElement src = (IHtmlSourceElement)picDiv
.QuerySelector(".left-wrap video > source");
if ((src == null && img == null) || a == null)
{
throw new NotFoundException(
"Either \"img\", \"source\" or \"a\" tag could not be found");
}
MemeInfo meme;
if (src != null)
{
meme = new MemeInfo
{
ViewURI = a.Href,
URI = src.Source,
Alt = string.Empty,
Name = a.TextContent,
Type = MediaType.Video
};
}
else if (input != null)
{
meme = new MemeInfo
{
ViewURI = a.Href,
URI = input.Value,
Alt = img.AlternativeText,
Name = a.TextContent,
Type = MediaType.Gif
};
}
else
{
meme = new MemeInfo
{
ViewURI = a.Href,
URI = img.Source,
Alt = img.AlternativeText,
Name = a.TextContent,
Type = MediaType.Image
};
}
return(meme);
}
