public void Serialize_HiddenTrueOnly()
{
var options = new WaitForSelectorOptions {
Hidden = true
};
var json = ClSerializer.Serialize(options);
Assert.AreEqual("{\"hidden\":true}", json);
}
public void Serialize_VisibleFalseOnly()
{
var options = new WaitForSelectorOptions {
Visible = false
};
var json = ClSerializer.Serialize(options);
Assert.AreEqual("{\"visible\":false}", json);
}
public override async Task <object> ParseAsync(CancellationToken cancellationToken)
{
using (var scope = serviceScopeFactory.CreateScope())
{
var raceDB = scope.ServiceProvider.GetService <RaceDB.Models.RaceDBContext>();
var page = await _engine.newPage();
await page.GoToAsync(_settings.Bet365.Url.MainPage.ToString());
var waitOption = new WaitForSelectorOptions
{
Timeout = 30000,
Hidden = true
};
var preLoadOuter = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.PreLoader, waitOption);
waitOption.Hidden = false;
var selectSport = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.Soccer, waitOption);
Thread.Sleep(3000);
await selectSport.ClickAsync();
Thread.Sleep(3000);
var totalCatergoryHtml = await page.GetContentAsync();
var attrs = HtmlHandler.GetImplement("TotalCatergory", totalCatergoryHtml).GetsAttributes(_totalCounrtyFilter);
var values = HtmlHandler.GetImplement("TotalCatergory", totalCatergoryHtml).Gets(_totalCounrtyFilter);
var catergoryData = (from a in attrs
from b in values
where a.Key == b.Key
select new
{
CatergoryKey = a.Value.Where(x => x.Key == "data-sportskey").FirstOrDefault().Value,
CatergoryValue = b.Value
});
var differentDatas = catergoryData.Where(x => raceDB.Category.Any(g => g.CategoryName == x.CatergoryValue && g.CategoryKey != x.CatergoryKey)).ToList();
foreach (var data in differentDatas)
{
var category = raceDB.Category.Where(x => x.CategoryName == data.CatergoryValue).FirstOrDefault();
category.CategoryKey = data.CatergoryKey;
}
await raceDB.SaveChangesAsync();
}
return(null);
}
public static async Task WaitForAnySelectors(this Page page, WaitForSelectorOptions options = null, params string[] selectors)
{
while (selectors.Length > 0)
{
foreach (var sel in selectors)
{
var result = await page.QuerySelectorAsync(sel);
if (result != null)
{
return;
}
}
await Task.Delay(100);
}
}
public VkontakteBrw(bool mobile)
{
Mobile = mobile;
BaseUrl = mobile ? "https://m.vk.com" : "https://vk.com";
UserAgent = Mobile ?
"Mozilla / 5.0(Linux; Android 4.2.1; en - us; Nexus 5 Build / JOP40D) AppleWebKit / 535.19(KHTML, like Gecko; googleweblight) Chrome / 38.0.1025.166 Mobile Safari/ 535.19" :
"Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 74.0.3723.0 Safari / 537.36";
DefaultNavigationOptions = new NavigationOptions {
Timeout = 30000
};
DefaultWaitForSelectorOptions = new WaitForSelectorOptions {
Timeout = 10000
};
DefaultClickOptions = new ClickOptions {
Delay = new Random().Next(100, 200)
};
}
private async Task GetCommunityDetailsAsync(BrowserContext browserContext, VkBrwCommunity community)
{
Page communityPage = await browserContext.NewPageAsync();
////groupPage.EvaluateFunctionAsync($"() => window.open('{groupUrl}')").GetAwaiter().GetResult();
////Target newWindowTarget = browser.WaitForTargetAsync(target => target.Url == "https://www.example.com/").Result;
////Page newPage = newWindowTarget.PageAsync().Result;
try
{
await communityPage.GoToAsync(community.CommunityUrl);
}
catch (NavigationException)
{
return;
}
WaitForSelectorOptions waitSelectorOptions = new WaitForSelectorOptions {
Timeout = 10000
};
ElementHandle communityLoadedElement = await communityPage.WaitForSelectorAsync("div#page_wrap.scroll_fix_wrap._page_wrap", waitSelectorOptions);
if (communityLoadedElement != null)
{
ElementHandle communityBlockedElement = await communityPage.QuerySelectorAsync("div.groups_blocked");
if (communityBlockedElement != null)
{
community.Blocked = true;
}
else
{
community.Type = await GetCommunityTypeAsync(communityPage);
community.CommunityId = await GetCommunityIdAsync(communityPage, community.Type);
}
}
await communityPage.CloseAsync();
}
public static async Task PuppeteerWaitForElement(BotData data, FindElementBy findBy, string identifier, bool hidden = false, bool visible = true,
int timeout = 30000)
{
data.Logger.LogHeader();
var frame = GetFrame(data);
var options = new WaitForSelectorOptions {
Hidden = hidden, Visible = visible, Timeout = timeout
};
if (findBy == FindElementBy.XPath)
{
await frame.WaitForXPathAsync(identifier, options);
}
else
{
await frame.WaitForSelectorAsync(BuildSelector(findBy, identifier), options);
}
data.Logger.Log($"Waited for element with {findBy} {identifier}", LogColors.DarkSalmon);
}
public async Task <Page> GetMatchOddsAsync(Page page, Match match)
{
var waitUntil = new NavigationOptions();
waitUntil.WaitUntil = new WaitUntilNavigation[1];
waitUntil.WaitUntil.Append(WaitUntilNavigation.Networkidle2);
await page.GoToAsync(_settings.Bet365.Url.MainPage.ToString(), waitUntil);
var waitOption = new WaitForSelectorOptions
{
Timeout = 30000,
Hidden = true
};
var preLoadOuter = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.PreLoader, waitOption);
waitOption.Hidden = false;
var selectSport = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.Soccer, waitOption);
await Task.Delay(6000);
await selectSport.ClickAsync();
var selectLeague = await page.WaitForXPathAsync("//*[@data-sportskey='" + match.League.LeagueKey.Trim() + "']", waitOption);
await Task.Delay(5000);
await selectLeague.ClickAsync();
// var selectLeagueList = await page.WaitForXPathAsync("//h3[@data-sportskey='" + leagueKey.Trim() + "']", waitOption);
// Thread.Sleep(5000);
// await selectLeagueList.ClickAsync();
//Thread.Sleep(2000);
var selectMatch = await page.WaitForXPathAsync("//*[@data-fixtureid='" + match.MatchKey.Trim() + "']/div/div", waitOption);
await Task.Delay(5000);
await selectMatch.ClickAsync();
return(page);
}
internal static async Task <Cookie> GetCookieByAuthorizationAsync(string pathToCookieFile)
{
/* ОЧЕНЬ ВАЖНО!
* В DefaultViewPort стоит параметр null, при котором в развернутом виде КОМПЬЮТЕРНАЯ ВЕРСИЯ,
* а в окне достаточно маленького размера у некоторых сайтов МОБИЛЬНАЯ ВЕРСИЯ!
* Следовательно, испольнование CSS-селекторов может быть удачным, а может и нет,
* в зависимости от размера браузера.
* В headless-режиме окно маленького размера. */
using (var browser = await PuppeteerSharp.Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
DefaultViewport = null,
//Args = PuppeteerSharpLaunchArgs.args
}))
{
Stopwatch timer = new Stopwatch();
var p = await GetPage(browser, "about:blank");
/* 10 попыток подключения нужны, если сервер плохой, или интернета вдруг нету.*/
bool success = false;
int connectCount = 0;
timer.Start();
while (connectCount < 10 && !success)
{
try
{
await p.GoToAsync("https://dnevnik2.petersburgedu.ru");
success = true;
Log.Information("Подключение к сайту {Site}: успешно!", "https://dnevnik2.petersburgedu.ru");
Log.Information($"{timer.ElapsedMilliseconds}");
timer.Restart();
}
catch (PuppeteerException e)
{
Log.Error(e, "GoToAsync({Site} failed", "https://dnevnik2.petersburgedu.ru");
Log.Information("Подключение к сайту {Site}: Ошибка, повторяю...", "https://dnevnik2.petersburgedu.ru");
Log.Information("Попытка № {attempt}", connectCount + 1);
await System.Threading.Tasks.Task.Delay(3000);
connectCount++;
}
}
WaitForSelectorOptions WaitForSelectorTimeout = new WaitForSelectorOptions {
Timeout = DefaultTimeout
};
Log.Information($"DefaultTimeout: {p.DefaultTimeout}");
Log.Information($"DefaultNavigationTimeout: {p.DefaultNavigationTimeout}");
const string button = "body > app-root > n3-grid > app-login > div > div.notice > div > app-login-form > div > button";
await p.WaitForSelectorAsync(button, WaitForSelectorTimeout);
await System.Threading.Tasks.Task.Delay(10000);
await p.ClickAsync(button);
Log.Information("Первый клик {Button}: успешно!", "Войти с ЕСИА");
Log.Information($"{timer.ElapsedMilliseconds}");
timer.Restart();
p = await GetPage(browser, "https://esia.gosuslugi.ru");
Log.Information($"DefaultTimeout: {p.DefaultTimeout}");
Log.Information($"DefaultNavigationTimeout: {p.DefaultNavigationTimeout}");
// Авторизация
await p.WaitForSelectorAsync("#mobileOrEmail", WaitForSelectorTimeout);
await p.FocusAsync("#mobileOrEmail");
await p.Keyboard.TypeAsync(ConfigJson.Login);
Log.Information($"Login: {timer.ElapsedMilliseconds}");
timer.Restart();
await p.WaitForSelectorAsync("#password", WaitForSelectorTimeout);
await p.FocusAsync("#password");
await p.Keyboard.TypeAsync(ConfigJson.Password);
Log.Information($"Password: {timer.ElapsedMilliseconds}");
timer.Restart();
await p.WaitForSelectorAsync("#loginByPwdButton > span", WaitForSelectorTimeout);
await p.ClickAsync("#loginByPwdButton > span");
Log.Information($"ClickAuthorizationButton: {timer.ElapsedMilliseconds}");
Log.Information("Авторизация: успешно!");
timer.Stop();
/* Куки нужны для того, чтобы сайт меня опознал
* при отправке http-запроса на сервер эл. дневника */
// 10 попыток получения cookie.
Cookie cookie;
int count = 0;
int attempts = (DefaultTimeout / 1000);
do
{
if (count > attempts)
{
throw new Exception("Cookie X-JMT-Token is not present.");
}
await System.Threading.Tasks.Task.Delay(1000);
var cookies = await p.GetCookiesAsync();
cookie = cookies.Where(c => c.Name == "X-JWT-Token").Select(c => new Cookie(c.Name, c.Value)).SingleOrDefault();
count++;
}while (cookie == null || cookie.Value == "");
//Здесь и далее безголовый браузер уже не нужен
await browser.CloseAsync();
var cookieAsJson = JsonConvert.SerializeObject(cookie);
//await File.WriteAllTextAsync(pathToCookieFile, cookieAsJson);
var fm = new ClassLibrary.File_Manager();
fm.OpenFile(pathToCookieFile, "Write", cookieAsJson);
return(cookie);
}
}
/// <summary>
/// Waits for a XPath expression to be added to the DOM and returns an <see cref="ElementObject" />.
/// </summary>
/// <typeparam name="T">The type of <see cref="ElementObject" /></typeparam>
/// <param name="page">A <see cref="Page" /></param>
/// <param name="xpath">A XPath expression of an element to wait for</param>
/// <param name="options">Optional waiting parameters</param>
/// <returns>A task that resolves to the <see cref="ElementObject" />, when a element specified by xpath string is added to DOM.</returns>
/// <seealso cref="Page.WaitForXPathAsync(string, WaitForSelectorOptions)"/>
public static async Task <T> WaitForXPathAsync <T>(this Page page, string xpath, WaitForSelectorOptions options = null) where T : ElementObject
{
var result = await page.GuardFromNull().WaitForXPathAsync(xpath, options).ConfigureAwait(false);
return(ProxyFactory.ElementObject <T>(page, result));
}
public static Task <ElementHandle[]> WaitForAllSelectors(this Page page, WaitForSelectorOptions options = null, params string[] selectors)
{
return(Task.WhenAll(selectors.Select(t => page.WaitForSelectorAsync(t, options)).ToArray()));
}
public async Task <string> GetMatchListAsync(string leagueKey)
{
var browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
ExecutablePath = "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
Headless = false
});
var page = await browser.NewPageAsync();
DeviceDescriptor IPhone = DeviceDescriptors.Get(DeviceDescriptorName.IPhone6);
var dic = new Dictionary <string, string>();
dic.Add("Referer", _settings.Bet365.Url.MainPage.ToString());
dic.Add("Accept-Encoding", "gzip, deflate, br");
dic.Add("Accept-Language", "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6");
dic.Add("Connection", "keep-alive");
await page.EmulateAsync(IPhone);
await page.SetRequestInterceptionAsync(true);
await page.SetExtraHttpHeadersAsync(dic);
page.Request += async(sender, e) =>
{
if (e.Request.ResourceType == ResourceType.Image)
{
await e.Request.AbortAsync();
}
else
{
await e.Request.ContinueAsync();
}
};
var waitUntil = new NavigationOptions();
waitUntil.WaitUntil = new WaitUntilNavigation[1];
waitUntil.WaitUntil.Append(WaitUntilNavigation.Networkidle2);
await page.GoToAsync(_settings.Bet365.Url.MainPage.ToString(), waitUntil);
var waitOption = new WaitForSelectorOptions
{
Timeout = 20000,
Hidden = true
};
var preLoadOuter = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.PreLoader, waitOption);
waitOption.Hidden = false;
var selectSport = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.Soccer, waitOption);
Thread.Sleep(5000);
await selectSport.ClickAsync();
var selectLeague = await page.WaitForXPathAsync("//*[@data-sportskey='" + leagueKey.Trim() + "']", waitOption);
Thread.Sleep(1000);
await selectLeague.ClickAsync();
//Thread.Sleep(2000);
var selectMatch = await page.WaitForXPathAsync("//*[@data-fixtureid]", waitOption);
// await selectMatch.ClickAsync();
return(await page.GetContentAsync());
}
public override async Task <object> ParseAsync(CancellationToken cancellationToken)
{
using (var scope = serviceScopeFactory.CreateScope())
{
var raceDB = scope.ServiceProvider.GetService <RaceDB.Models.RaceDBContext>();
var categoryKeys = raceDB.Category.Where(x => x.Status == 1).ToList();
foreach (var key in categoryKeys)
{
Func <XPathHandler, XPathHandler> categoryFilter = element => element.Find(TagEnum.Div, "@class='eventWrapper cc_34_8'");
var page = await _engine.newPage();
await page.GoToAsync(_settings.Bet365.Url.MainPage.ToString());
var waitOption = new WaitForSelectorOptions
{
Timeout = 30000,
Hidden = true
};
var preLoadOuter = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.PreLoader, waitOption);
waitOption.Hidden = false;
var selectSport = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.Soccer, waitOption);
Thread.Sleep(3000);
await selectSport.ClickAsync();
Thread.Sleep(3000);
var totalLeagueHtml = await page.GetContentAsync();
var html = HtmlHandler.GetImplement("TotalLeague", totalLeagueHtml).Gets(categoryFilter)[0];
var attrs = HtmlHandler.GetImplement("TotalLeague", html).GetsAttributes(_totalLeagueFilter);
var values = HtmlHandler.GetImplement("TotalLeague", html).Gets(_totalLeagueValueFilter);
var leagueData = (from a in attrs
from b in values
where a.Key == b.Key
select new
{
LeagueKey = a.Value.Where(x => x.Key == "data-sportskey").FirstOrDefault().Value,
LeagueValue = b.Value
});
var differentDatas = leagueData.Where(x => raceDB.League.Any(g => g.LeagueName != x.LeagueValue && g.LeagueKey != x.LeagueKey)).ToList();
foreach (var data in differentDatas)
{
var league = raceDB.League.Where(x => x.LeagueName == data.LeagueValue).FirstOrDefault();
if (league != null)
{
league.LeagueKey = data.LeagueKey;
}
else
{
RaceDB.Models.League leagueModel = new RaceDB.Models.League();
leagueModel.CategoryId = key.CategoryId;
leagueModel.LeagueKey = data.LeagueKey;
leagueModel.LeagueName = data.LeagueValue;
raceDB.Add(leagueModel);
}
//.LeagueKey = data.LeagueKey;
}
await raceDB.SaveChangesAsync();
}
}
return(null);
}
