private async Task <string> GetCommunityIdAsync(Page communityPage, CommunityTypes type)
{
string result = null;
string linkString = (type == CommunityTypes.Public) ?
"#public_followers.module.clear.people_module._module" : "#group_followers.module.clear.people_module._module";
ElementHandle communityFollowersElement = await communityPage.QuerySelectorAsync(linkString);
if (communityFollowersElement != null)
{
ElementHandle communityFollowersSearchElement = await communityFollowersElement.QuerySelectorAsync("a.module_header");
string communityFollowersSearchLink = await communityFollowersSearchElement.EvaluateFunctionAsync <string>("('a', a => a.href)");
string linkLastPart = communityFollowersSearchLink.Split("=").Last();
if (!string.IsNullOrEmpty(linkLastPart))
{
result = linkLastPart;
}
}
else
{
bool isPersonalPage = await IsPersonalPage(communityPage);
if (isPersonalPage)
{
//создать пользователя
}
}
return(result);
}
private static async Task AssertHoverAsync(ElementHandle tabHeader)
{
await tabHeader.HoverAsync();
var textColor = await tabHeader.EvaluateFunctionAsync <string>("x=>window.getComputedStyle(x,null).color");
Assert.Equal("rgb(64, 158, 255)", textColor);
}
private static async Task AssertErrorAsync(bool showRequired, ElementHandle error, string expectedRrrorText)
{
if (showRequired)
{
Assert.NotNull(error);
var errorText = await error.EvaluateFunctionAsync <string>("x=>x.innerText");
Assert.Equal(expectedRrrorText, errorText.Trim());
}
}
private async Task AssertBodyAsync(ElementHandle body, string text)
{
if (text != "Component")
{
var bodyText = (await body.EvaluateFunctionAsync <string>("x=>x.innerText")).Trim();
if (text == "选项1")
{
Assert.Equal("内容1", bodyText);
}
else if (text == "卡2")
{
Assert.Equal("内容2", bodyText);
}
else if (text == "卡3")
{
Assert.Equal("内容3", bodyText);
}
else if (text == "标题0")
{
Assert.Equal("内容0", bodyText);
}
else if (text == "标题1")
{
Assert.Equal("内容1", bodyText);
}
else if (text == "标题2")
{
Assert.Equal("内容2", bodyText);
}
else if (text == "标题3")
{
Assert.Equal("内容3", bodyText);
}
else if (text == "标题3")
{
Assert.Equal("内容3", bodyText);
}
else if (text == "标题4")
{
Assert.Equal("内容4", bodyText);
}
else if (text == "标题5")
{
Assert.Equal("内容5", bodyText);
}
}
else
{
var input = await body.QuerySelectorAsync("label.el-checkbox > span.el-checkbox__input > input");
Assert.NotNull(input);
}
}
private static async Task AssertCheckBoxButtonAsync(bool isChecked, ElementHandle checkbox)
{
var cls = await checkbox.EvaluateFunctionAsync <string>("x=>x.className");
ElementHandle el;
if (isChecked)
{
Assert.Equal("el-checkbox-button is-checked ", cls);
}
else
{
Assert.Equal("el-checkbox-button ", cls);
}
el = await checkbox.QuerySelectorAsync("span.el-checkbox-button__inner");
Assert.NotNull(el);
el = await checkbox.QuerySelectorAsync("input[type='checkbox'].el-checkbox-button__original");
Assert.NotNull(el);
}
private async Task <VkBrwCommunity> GetCommunityAsync(ElementHandle communityElement, BrowserContext userContext)
{
ElementHandle communityNameElement = await communityElement.QuerySelectorAsync("div.fans_idol_name");
if (communityNameElement != null)
{
VkBrwCommunity nwPage = new VkBrwCommunity();
////await page.EvaluateFunctionAsync(@"() => {
//// Array.from(document.querySelectorAll('li'))
//// .find(l => l.querySelector('span').innerText === 'fire').querySelector('INPUT').click();
////}");
// equals to $eval('a', a => a.innerText)
nwPage.Name = await communityNameElement.EvaluateFunctionAsync <string>("('a', a => a.innerText)");
ElementHandle communityLinkElement = await communityElement.QuerySelectorAsync("a.fans_idol_lnk");
nwPage.CommunityUrl = await communityLinkElement.EvaluateFunctionAsync <string>("('a', a => a.href)");
ElementHandle communityStatusElement = await communityElement.QuerySelectorAsync("div.fans_idol_status");
nwPage.Status = await communityStatusElement.EvaluateFunctionAsync <string>("('div', div => div.innerText)");
ElementHandle communitySizeElement = await communityElement.QuerySelectorAsync("div.fans_idol_size");
nwPage.Size = await communitySizeElement.EvaluateFunctionAsync <string>("('div', div => div.innerText)");
if (!nwPage.CommunityUrl.Contains("event"))
{
await GetCommunityDetailsAsync(userContext, nwPage);
}
return(nwPage);
}
else
{
return(null);
}
}
private async Task AssertBodyAsync(ElementHandle body, string text)
{
var bodyText = await body.EvaluateFunctionAsync <string>("x=>x.innerText");
Assert.Equal(text, bodyText?.Trim());
}
protected async Task <string> ExtractPropertyAsync(ElementHandle handle, string nodeProperty)
{
return(await handle.EvaluateFunctionAsync <string>($"t => t.{nodeProperty}"));
}
public async Task ProcessAsync(MessageContext context, HttpApiClient api)
{
using (var page = await Chrome.OpenNewPageAsync())
{
await page.SetViewportAsync(new ViewPortOptions
{
DeviceScaleFactor = 2,
Width = 400,
Height = 3500,
});
await page.GoToAsync("https://qieman.com/idx-eval");
//// Get update time.
//const string updateDateSelector = "#app > div > div.ant-layout-container > div > div > div.ant-col-xs-24.ant-col-lg-15 > div > div > div.sc-jWBwVP.dtKXxN > div > span.qm-header-note > div > p";
//var updateDateElement = await page.WaitForSelectorAsync(updateDateSelector);
//if (updateDateElement != null)
//{
// var property = await updateDateElement.GetPropertyAsync("textContent");
// if (property != null)
// {
// var textContent = (await property.JsonValueAsync()).ToString();
// await api.SendMessageAsync(context.Endpoint, textContent);
// }
// else
// {
// Logger.Debug($"{nameof(property)} is null.");
// }
//}
//else
//{
// Logger.Debug($"{nameof(updateDateElement)} is null.");
//}
//await Task.Delay(TimeSpan.FromSeconds(3));
//var xp = await page.QuerySelectorAsync("#app > div > div.ant-layout-container > div > div > div.ant-col-xs-24.ant-col-lg-15 > div > div > div.flex-table__67b31");
//if (xp != null)
// await api.SendMessageAsync(context.Endpoint, Message.ByteArrayImage(await xp.ScreenshotDataAsync(new ScreenshotOptions
// {
// })));
const string detailSelector = "#app > div.ant-layout-main.ant-layout-main-f > div.ant-layout-container.safe-padding-bottom > div > div > div > div.index__29fc4";
ElementHandle detailElement = await page.WaitForSelectorAsync(detailSelector).ConfigureAwait(false);
// delete advertisement
const string adSelector = "#app > div > div.ant-layout-container > div > div > div > div.index__29fc4 > a > img";
ElementHandle adElement = await page.QuerySelectorAsync(adSelector).ConfigureAwait(false);
if (!(adElement is null))
{
await adElement.EvaluateFunctionAsync(@"(element) => { element.parentElement.remove(); }").ConfigureAwait(false);
}
var data = await detailElement
.ScreenshotDataAsync(new ScreenshotOptions
{
//FullPage = true,
});
bool inLoop = true;
int retry = 3;
do
{
var mesResponse = await api.SendMessageAsync(context.Endpoint, Message.ByteArrayImage(data)).ConfigureAwait(false);
inLoop = mesResponse == null;
} while (inLoop && --retry > 0);
}
}
public async Task <List <VkBrwCommunity> > GetCommunities(BrowserContext userContext, Page userPage, VkBrwUser user)
{
List <VkBrwCommunity> nwPages = new List <VkBrwCommunity>();
await userPage.GoToAsync(user.ProfileLink);
ElementHandle userIsLoaded = await userPage.WaitForSelectorAsync("#page_info_wrap.page_info_wrap");
ElementHandle[] hiddenProfileBlock = await userPage.QuerySelectorAllAsync("h5.profile_deleted_text");
if (hiddenProfileBlock.Length == 0)
{
ElementHandle idolsLoaded = await userPage.WaitForSelectorAsync("#profile_idols");
// Интересные страницы
ElementHandle[] noteworthyPagesBlock = await userPage.QuerySelectorAllAsync("#profile_idols.module.clear.page_list_module._module");
if (noteworthyPagesBlock.Length == 1)
{
ElementHandle noteworthyPages = noteworthyPagesBlock.First();
ElementHandle[] noteworthyPagesHeaderBlock = await noteworthyPages.QuerySelectorAllAsync("a.module_header");
if (noteworthyPagesHeaderBlock.Length == 1)
{
ClickOptions clickOptions = new ClickOptions {
Delay = new Random().Next(30, 100)
};
ElementHandle noteworthyPagesLinkElement = noteworthyPagesHeaderBlock.First();
await noteworthyPagesLinkElement.ClickAsync(clickOptions);
ElementHandle noteworthyPagesIsOpened = await userPage.WaitForSelectorAsync("#fans_rowsidols.fans_rows.fans_idols");
ElementHandle[] closeBlock = await userPage.QuerySelectorAllAsync("div.box_x_button");
if (closeBlock.Length == 1)
{
ElementHandle[] pagesCountBlock = await userPage.QuerySelectorAllAsync("span.ui_box_header_cnt");
if (pagesCountBlock.Length == 1)
{
ElementHandle pagesTotalCountElement = pagesCountBlock.First();
string pagesTotalCountValue = await pagesTotalCountElement.EvaluateFunctionAsync <string>("('span', span => span.innerText)");
if (!string.IsNullOrEmpty(pagesTotalCountValue))
{
bool pagesTotalCountReceived = int.TryParse(pagesTotalCountValue, out int pagesTotalCount);
if (pagesTotalCountReceived && pagesTotalCount > 0)
{
ElementHandle[] pagesVisibleElements = await userPage.QuerySelectorAllAsync("div.fans_idol_row.inl_bl");
if (pagesVisibleElements.Length < pagesTotalCount)
{
PressOptions pressOptions = new PressOptions {
Delay = new Random().Next(20, 40)
};
await userPage.FocusAsync("input");
await userPage.Keyboard.PressAsync("Tab", pressOptions);
int visiblePagesCounter = pagesVisibleElements.Length;
while (visiblePagesCounter < pagesTotalCount)
{
await userPage.Keyboard.PressAsync("PageDown", pressOptions);
await Task.Delay(new Random().Next(250, 350));
await userPage.Keyboard.PressAsync("PageDown", pressOptions);
await Task.Delay(new Random().Next(250, 350));
await userPage.Keyboard.PressAsync("PageDown", pressOptions);
await Task.Delay(new Random().Next(250, 350));
await userPage.Keyboard.PressAsync("PageDown", pressOptions);
await Task.Delay(new Random().Next(250, 350));
ElementHandle[] newPagesVisibleElements = await userPage.QuerySelectorAllAsync("div.fans_idol_row.inl_bl");
if (newPagesVisibleElements.Length == visiblePagesCounter)
{
break;
}
visiblePagesCounter = newPagesVisibleElements.Length;
}
}
ElementHandle[] nwPagesElements = await userPage.QuerySelectorAllAsync("div.fans_idol_info");
foreach (var element in nwPagesElements)
{
VkBrwCommunity community = await GetCommunityAsync(element, userContext);
if (community != null)
{
nwPages.Add(community);
community.Users.Add(user);
}
}
}
}
}
ElementHandle closeButtonElement = closeBlock.First();
await closeButtonElement.ClickAsync(clickOptions);
}
}
}
}
else
{
user.HiddenProfile = true;
ElementHandle[] pageNameElements = await userPage.QuerySelectorAllAsync("h2.page_name");
if (pageNameElements.Length == 1)
{
var pageElement = pageNameElements.First();
user.PageName = await pageElement.EvaluateFunctionAsync <string>("('h2', h2 => h2.innerText)");
}
}
return(nwPages);
}
// Total CPU [unit, %]: 676
private async Task <List <VkBrwUserWallPost> > GetPostsViaDesktopPuppeteer()
{
List <VkBrwUserWallPost> result = new List <VkBrwUserWallPost>();
string vkMessageHeaderInformationLocalized = (Vkontakte.Language == BrowsingLanguage.RU) ?
"Информация" :
"Information";
string vkMessageHeaderErrorLocalized = (Vkontakte.Language == BrowsingLanguage.RU) ?
"Ошибка" :
"Error";
string hiddenDescriptionLocalized = (Vkontakte.Language == BrowsingLanguage.RU) ?
"Пользователь предпочёл скрыть эту страницу." :
"This user has chosen to hide their page.";
ElementHandle vkMessageElement = await _userPage.QuerySelectorAsync("div.message_page.page_block");
if (vkMessageElement != null)
{
ElementHandle vkMessageHeaderElement = await vkMessageElement.QuerySelectorAsync("div.message_page_title");
if (vkMessageHeaderElement != null)
{
string vkMessageHeaderText = await vkMessageHeaderElement.EvaluateFunctionAsync <string>("('div', div => div.innerText)");
if (vkMessageHeaderText == vkMessageHeaderInformationLocalized)
{
Vkontakte.NonExistingProfiles += 1;
}
else if (vkMessageHeaderText == vkMessageHeaderErrorLocalized)
{
ElementHandle vkMessageBodyElement = await vkMessageElement.QuerySelectorAsync("div.message_page_body");
if (vkMessageBodyElement != null)
{
string vkMessageBodyText = await vkMessageBodyElement.EvaluateFunctionAsync <string>("('div', div => div.innerText)");
string vkMessageBodyDescription = vkMessageBodyText.Split('\n')[0];
if (vkMessageBodyDescription == hiddenDescriptionLocalized)
{
HiddenProfile = true;
Vkontakte.NonPublicProfiles += 1;
}
else// if (mmm == "Вы попытались загрузить более одной однотипной страницы в секунду.")
{
Console.WriteLine(ProfileLink + " has the following vk message: " + vkMessageBodyDescription);
}
}
else
{
Console.WriteLine(ProfileLink + " doesn't contain vk message body.");
}
}
else
{
Console.WriteLine(ProfileLink + " error header text is " + vkMessageHeaderText);
}
}
}
else
{
ElementHandle[] wallPostElements = await _userPage.QuerySelectorAllAsync("div._post_content");
if (wallPostElements != null && wallPostElements.Length > 0)
{
foreach (ElementHandle postElement in wallPostElements)
{
VkBrwUserWallPost post = new VkBrwUserWallPost();
ElementHandle dateElement = await postElement.QuerySelectorAsync("span.rel_date");
if (dateElement != null)
{
string postDateRaw = await dateElement.EvaluateFunctionAsync <string>("('span', span => span.innerText)");
DateTime postDate = Vkontakte.Language == BrowsingLanguage.RU
? ConvertVkWallPostRusDate(postDateRaw)
: ConvertVkWallPostEngDate(postDateRaw);
if (postDate != DateTime.MinValue)
{
post.Date = postDate;
}
}
ElementHandle wallPostTextElement = await postElement.QuerySelectorAsync("div.wall_text");
if (wallPostTextElement != null)
{
post.Repost = (await wallPostTextElement.QuerySelectorAsync("div.copy_quote") != null);
ElementHandle postT = await wallPostTextElement.QuerySelectorAsync("div.wall_post_text");
if (postT != null)
{
post.Text = await postT.EvaluateFunctionAsync <string>("'div', div => div.innerText");
}
}
result.Add(post);
post.User = this;
WallPosts.Add(post);
}
Vkontakte.CollectedProfiles += 1;
}
else
{
Vkontakte.EmptyWallProfiles += 1;
}
}
return(result);
}
private async Task <List <VkBrwUserWallPost> > GetPostsViaMobilePuppeteer()
{
List <VkBrwUserWallPost> result = new List <VkBrwUserWallPost>();
string hiddenDescriptionLocalized = (Vkontakte.Language == BrowsingLanguage.RU) ?
"Страница доступна только авторизованным пользователям." :
"You have to log in to view this page.";
string emptyWallDescriptionLocalized = (Vkontakte.Language == BrowsingLanguage.RU) ?
"На стене пока нет ни одной записи." :
"This wall is empty.";
ElementHandle vkMessageElement = await _userPage.QuerySelectorAsync("div.service_msg.service_msg_null");
if (vkMessageElement != null)
{
//"Страница удалена либо ещё не создана."
//
string vkMessageText = await vkMessageElement.EvaluateFunctionAsync <string>("('div', div => div.innerText)");
if (vkMessageText == hiddenDescriptionLocalized)
{
Vkontakte.NonPublicProfiles += 1;
}
else if (vkMessageText == emptyWallDescriptionLocalized)
{
Vkontakte.EmptyWallProfiles += 1;
}
else
{
Vkontakte.NonExistingProfiles += 1;
}
}
else
{
ElementHandle[] wallPostElements = await _userPage.QuerySelectorAllAsync("div.wall_item");
if (wallPostElements.Length > 0)
{
foreach (ElementHandle postElement in wallPostElements)
{
VkBrwUserWallPost post = new VkBrwUserWallPost();
ElementHandle dateElement = await postElement.QuerySelectorAsync("a.wi_date");
if (dateElement != null)
{
string postDateRaw = await dateElement.EvaluateFunctionAsync <string>("('a', a => a.innerText)");
DateTime postDate = Vkontakte.Language == BrowsingLanguage.RU
? ConvertVkWallPostRusDate(postDateRaw)
: ConvertVkWallPostEngDate(postDateRaw);
if (postDate != DateTime.MinValue)
{
post.Date = postDate;
}
}
ElementHandle wallPostTextElement = await postElement.QuerySelectorAsync("div.pi_text");
if (wallPostTextElement != null)
{
post.Text = await wallPostTextElement.EvaluateFunctionAsync <string>("'span', span => span.innerText");
}
string repostValue = await postElement.EvaluateFunctionAsync <string>("('div', div => { return div.getAttribute('data-copy'); })");
post.Repost = !string.IsNullOrEmpty(repostValue);
result.Add(post);
post.User = this;
WallPosts.Add(post);
}
Vkontakte.CollectedProfiles += 1;
}
else
{
Vkontakte.NonPublicProfiles += 1;
}
}
return(result);
}
