Exemple #1
0
        public void Serialize_HiddenTrueOnly()
        {
            var options = new WaitForSelectorOptions {
                Hidden = true
            };
            var json = ClSerializer.Serialize(options);

            Assert.AreEqual("{\"hidden\":true}", json);
        }
Exemple #2
0
        public void Serialize_VisibleFalseOnly()
        {
            var options = new WaitForSelectorOptions {
                Visible = false
            };
            var json = ClSerializer.Serialize(options);

            Assert.AreEqual("{\"visible\":false}", json);
        }
Exemple #3
0
        public override async Task <object> ParseAsync(CancellationToken cancellationToken)
        {
            using (var scope = serviceScopeFactory.CreateScope())
            {
                var raceDB = scope.ServiceProvider.GetService <RaceDB.Models.RaceDBContext>();
                var page   = await _engine.newPage();

                await page.GoToAsync(_settings.Bet365.Url.MainPage.ToString());

                var waitOption = new WaitForSelectorOptions
                {
                    Timeout = 30000,
                    Hidden  = true
                };
                var preLoadOuter = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.PreLoader, waitOption);

                waitOption.Hidden = false;
                var selectSport = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.Soccer, waitOption);

                Thread.Sleep(3000);
                await selectSport.ClickAsync();

                Thread.Sleep(3000);

                var totalCatergoryHtml = await page.GetContentAsync();

                var attrs         = HtmlHandler.GetImplement("TotalCatergory", totalCatergoryHtml).GetsAttributes(_totalCounrtyFilter);
                var values        = HtmlHandler.GetImplement("TotalCatergory", totalCatergoryHtml).Gets(_totalCounrtyFilter);
                var catergoryData = (from a in attrs
                                     from b in values
                                     where a.Key == b.Key
                                     select new
                {
                    CatergoryKey = a.Value.Where(x => x.Key == "data-sportskey").FirstOrDefault().Value,
                    CatergoryValue = b.Value
                });

                var differentDatas = catergoryData.Where(x => raceDB.Category.Any(g => g.CategoryName == x.CatergoryValue && g.CategoryKey != x.CatergoryKey)).ToList();

                foreach (var data in differentDatas)
                {
                    var category = raceDB.Category.Where(x => x.CategoryName == data.CatergoryValue).FirstOrDefault();
                    category.CategoryKey = data.CatergoryKey;
                }
                await raceDB.SaveChangesAsync();
            }
            return(null);
        }
Exemple #4
0
        public static async Task WaitForAnySelectors(this Page page, WaitForSelectorOptions options = null, params string[] selectors)
        {
            while (selectors.Length > 0)
            {
                foreach (var sel in selectors)
                {
                    var result = await page.QuerySelectorAsync(sel);

                    if (result != null)
                    {
                        return;
                    }
                }
                await Task.Delay(100);
            }
        }
Exemple #5
0
        public VkontakteBrw(bool mobile)
        {
            Mobile    = mobile;
            BaseUrl   = mobile ? "https://m.vk.com" : "https://vk.com";
            UserAgent = Mobile ?
                        "Mozilla / 5.0(Linux; Android 4.2.1; en - us; Nexus 5 Build / JOP40D) AppleWebKit / 535.19(KHTML, like Gecko; googleweblight) Chrome / 38.0.1025.166 Mobile Safari/ 535.19" :
                        "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 74.0.3723.0 Safari / 537.36";

            DefaultNavigationOptions = new NavigationOptions {
                Timeout = 30000
            };
            DefaultWaitForSelectorOptions = new WaitForSelectorOptions {
                Timeout = 10000
            };
            DefaultClickOptions = new ClickOptions {
                Delay = new Random().Next(100, 200)
            };
        }
Exemple #6
0
        private async Task GetCommunityDetailsAsync(BrowserContext browserContext, VkBrwCommunity community)
        {
            Page communityPage = await browserContext.NewPageAsync();

            ////groupPage.EvaluateFunctionAsync($"() => window.open('{groupUrl}')").GetAwaiter().GetResult();
            ////Target newWindowTarget = browser.WaitForTargetAsync(target => target.Url == "https://www.example.com/").Result;
            ////Page newPage = newWindowTarget.PageAsync().Result;

            try
            {
                await communityPage.GoToAsync(community.CommunityUrl);
            }
            catch (NavigationException)
            {
                return;
            }

            WaitForSelectorOptions waitSelectorOptions = new WaitForSelectorOptions {
                Timeout = 10000
            };

            ElementHandle communityLoadedElement = await communityPage.WaitForSelectorAsync("div#page_wrap.scroll_fix_wrap._page_wrap", waitSelectorOptions);

            if (communityLoadedElement != null)
            {
                ElementHandle communityBlockedElement = await communityPage.QuerySelectorAsync("div.groups_blocked");

                if (communityBlockedElement != null)
                {
                    community.Blocked = true;
                }
                else
                {
                    community.Type = await GetCommunityTypeAsync(communityPage);

                    community.CommunityId = await GetCommunityIdAsync(communityPage, community.Type);
                }
            }

            await communityPage.CloseAsync();
        }
Exemple #7
0
        public static async Task PuppeteerWaitForElement(BotData data, FindElementBy findBy, string identifier, bool hidden = false, bool visible = true,
                                                         int timeout = 30000)
        {
            data.Logger.LogHeader();

            var frame   = GetFrame(data);
            var options = new WaitForSelectorOptions {
                Hidden = hidden, Visible = visible, Timeout = timeout
            };

            if (findBy == FindElementBy.XPath)
            {
                await frame.WaitForXPathAsync(identifier, options);
            }
            else
            {
                await frame.WaitForSelectorAsync(BuildSelector(findBy, identifier), options);
            }

            data.Logger.Log($"Waited for element with {findBy} {identifier}", LogColors.DarkSalmon);
        }
        public async Task <Page> GetMatchOddsAsync(Page page, Match match)
        {
            var waitUntil = new NavigationOptions();

            waitUntil.WaitUntil = new WaitUntilNavigation[1];
            waitUntil.WaitUntil.Append(WaitUntilNavigation.Networkidle2);
            await page.GoToAsync(_settings.Bet365.Url.MainPage.ToString(), waitUntil);

            var waitOption = new WaitForSelectorOptions
            {
                Timeout = 30000,
                Hidden  = true
            };
            var preLoadOuter = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.PreLoader, waitOption);

            waitOption.Hidden = false;
            var selectSport = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.Soccer, waitOption);

            await Task.Delay(6000);

            await selectSport.ClickAsync();

            var selectLeague = await page.WaitForXPathAsync("//*[@data-sportskey='" + match.League.LeagueKey.Trim() + "']", waitOption);

            await Task.Delay(5000);

            await selectLeague.ClickAsync();

            // var selectLeagueList = await page.WaitForXPathAsync("//h3[@data-sportskey='" + leagueKey.Trim() + "']", waitOption);
            // Thread.Sleep(5000);
            // await selectLeagueList.ClickAsync();
            //Thread.Sleep(2000);
            var selectMatch = await page.WaitForXPathAsync("//*[@data-fixtureid='" + match.MatchKey.Trim() + "']/div/div", waitOption);

            await Task.Delay(5000);

            await selectMatch.ClickAsync();

            return(page);
        }
Exemple #9
0
        internal static async Task <Cookie> GetCookieByAuthorizationAsync(string pathToCookieFile)
        {
            /* ОЧЕНЬ ВАЖНО!
             * В DefaultViewPort стоит параметр null, при котором в развернутом виде КОМПЬЮТЕРНАЯ ВЕРСИЯ,
             * а в окне достаточно маленького размера у некоторых сайтов МОБИЛЬНАЯ ВЕРСИЯ!
             * Следовательно, испольнование CSS-селекторов может быть удачным, а может и нет,
             * в зависимости от размера браузера.
             * В headless-режиме окно маленького размера. */
            using (var browser = await PuppeteerSharp.Puppeteer.LaunchAsync(new LaunchOptions
            {
                Headless = true,
                DefaultViewport = null,
                //Args = PuppeteerSharpLaunchArgs.args
            }))
            {
                Stopwatch timer = new Stopwatch();
                var       p     = await GetPage(browser, "about:blank");

                /* 10 попыток подключения нужны, если сервер плохой, или интернета вдруг нету.*/

                bool success      = false;
                int  connectCount = 0;
                timer.Start();
                while (connectCount < 10 && !success)
                {
                    try
                    {
                        await p.GoToAsync("https://dnevnik2.petersburgedu.ru");

                        success = true;
                        Log.Information("Подключение к сайту {Site}: успешно!", "https://dnevnik2.petersburgedu.ru");
                        Log.Information($"{timer.ElapsedMilliseconds}");
                        timer.Restart();
                    }
                    catch (PuppeteerException e)
                    {
                        Log.Error(e, "GoToAsync({Site} failed", "https://dnevnik2.petersburgedu.ru");
                        Log.Information("Подключение к сайту {Site}: Ошибка, повторяю...", "https://dnevnik2.petersburgedu.ru");
                        Log.Information("Попытка № {attempt}", connectCount + 1);
                        await System.Threading.Tasks.Task.Delay(3000);

                        connectCount++;
                    }
                }


                WaitForSelectorOptions WaitForSelectorTimeout = new WaitForSelectorOptions {
                    Timeout = DefaultTimeout
                };

                Log.Information($"DefaultTimeout: {p.DefaultTimeout}");
                Log.Information($"DefaultNavigationTimeout: {p.DefaultNavigationTimeout}");

                const string button = "body > app-root > n3-grid > app-login > div > div.notice > div > app-login-form > div > button";
                await p.WaitForSelectorAsync(button, WaitForSelectorTimeout);

                await System.Threading.Tasks.Task.Delay(10000);

                await p.ClickAsync(button);

                Log.Information("Первый клик {Button}: успешно!", "Войти с ЕСИА");
                Log.Information($"{timer.ElapsedMilliseconds}");
                timer.Restart();

                p = await GetPage(browser, "https://esia.gosuslugi.ru");

                Log.Information($"DefaultTimeout: {p.DefaultTimeout}");
                Log.Information($"DefaultNavigationTimeout: {p.DefaultNavigationTimeout}");

                // Авторизация
                await p.WaitForSelectorAsync("#mobileOrEmail", WaitForSelectorTimeout);

                await p.FocusAsync("#mobileOrEmail");

                await p.Keyboard.TypeAsync(ConfigJson.Login);

                Log.Information($"Login: {timer.ElapsedMilliseconds}");
                timer.Restart();

                await p.WaitForSelectorAsync("#password", WaitForSelectorTimeout);

                await p.FocusAsync("#password");

                await p.Keyboard.TypeAsync(ConfigJson.Password);

                Log.Information($"Password: {timer.ElapsedMilliseconds}");
                timer.Restart();

                await p.WaitForSelectorAsync("#loginByPwdButton > span", WaitForSelectorTimeout);

                await p.ClickAsync("#loginByPwdButton > span");

                Log.Information($"ClickAuthorizationButton: {timer.ElapsedMilliseconds}");

                Log.Information("Авторизация: успешно!");
                timer.Stop();

                /* Куки нужны для того, чтобы сайт меня опознал
                 * при отправке http-запроса на сервер эл. дневника */

                // 10 попыток получения cookie.
                Cookie cookie;
                int    count    = 0;
                int    attempts = (DefaultTimeout / 1000);
                do
                {
                    if (count > attempts)
                    {
                        throw new Exception("Cookie X-JMT-Token is not present.");
                    }
                    await System.Threading.Tasks.Task.Delay(1000);

                    var cookies = await p.GetCookiesAsync();

                    cookie = cookies.Where(c => c.Name == "X-JWT-Token").Select(c => new Cookie(c.Name, c.Value)).SingleOrDefault();
                    count++;
                }while (cookie == null || cookie.Value == "");

                //Здесь и далее безголовый браузер уже не нужен
                await browser.CloseAsync();

                var cookieAsJson = JsonConvert.SerializeObject(cookie);
                //await File.WriteAllTextAsync(pathToCookieFile, cookieAsJson);
                var fm = new ClassLibrary.File_Manager();
                fm.OpenFile(pathToCookieFile, "Write", cookieAsJson);
                return(cookie);
            }
        }
        /// <summary>
        /// Waits for a XPath expression to be added to the DOM and returns an <see cref="ElementObject" />.
        /// </summary>
        /// <typeparam name="T">The type of <see cref="ElementObject" /></typeparam>
        /// <param name="page">A <see cref="Page" /></param>
        /// <param name="xpath">A XPath expression of an element to wait for</param>
        /// <param name="options">Optional waiting parameters</param>
        /// <returns>A task that resolves to the <see cref="ElementObject" />, when a element specified by xpath string is added to DOM.</returns>
        /// <seealso cref="Page.WaitForXPathAsync(string, WaitForSelectorOptions)"/>
        public static async Task <T> WaitForXPathAsync <T>(this Page page, string xpath, WaitForSelectorOptions options = null) where T : ElementObject
        {
            var result = await page.GuardFromNull().WaitForXPathAsync(xpath, options).ConfigureAwait(false);

            return(ProxyFactory.ElementObject <T>(page, result));
        }
Exemple #11
0
 public static Task <ElementHandle[]> WaitForAllSelectors(this Page page, WaitForSelectorOptions options = null, params string[] selectors)
 {
     return(Task.WhenAll(selectors.Select(t => page.WaitForSelectorAsync(t, options)).ToArray()));
 }
Exemple #12
0
        public async Task <string> GetMatchListAsync(string leagueKey)
        {
            var browser = await Puppeteer.LaunchAsync(new LaunchOptions
            {
                ExecutablePath = "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
                Headless       = false
            });

            var page = await browser.NewPageAsync();

            DeviceDescriptor IPhone = DeviceDescriptors.Get(DeviceDescriptorName.IPhone6);
            var dic = new Dictionary <string, string>();

            dic.Add("Referer", _settings.Bet365.Url.MainPage.ToString());
            dic.Add("Accept-Encoding", "gzip, deflate, br");
            dic.Add("Accept-Language", "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6");
            dic.Add("Connection", "keep-alive");
            await page.EmulateAsync(IPhone);

            await page.SetRequestInterceptionAsync(true);

            await page.SetExtraHttpHeadersAsync(dic);

            page.Request += async(sender, e) =>
            {
                if (e.Request.ResourceType == ResourceType.Image)
                {
                    await e.Request.AbortAsync();
                }
                else
                {
                    await e.Request.ContinueAsync();
                }
            };
            var waitUntil = new NavigationOptions();

            waitUntil.WaitUntil = new WaitUntilNavigation[1];
            waitUntil.WaitUntil.Append(WaitUntilNavigation.Networkidle2);
            await page.GoToAsync(_settings.Bet365.Url.MainPage.ToString(), waitUntil);

            var waitOption = new WaitForSelectorOptions
            {
                Timeout = 20000,
                Hidden  = true
            };
            var preLoadOuter = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.PreLoader, waitOption);

            waitOption.Hidden = false;
            var selectSport = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.Soccer, waitOption);

            Thread.Sleep(5000);
            await selectSport.ClickAsync();

            var selectLeague = await page.WaitForXPathAsync("//*[@data-sportskey='" + leagueKey.Trim() + "']", waitOption);

            Thread.Sleep(1000);
            await selectLeague.ClickAsync();

            //Thread.Sleep(2000);
            var selectMatch = await page.WaitForXPathAsync("//*[@data-fixtureid]", waitOption);

            // await selectMatch.ClickAsync();
            return(await page.GetContentAsync());
        }
        public override async Task <object> ParseAsync(CancellationToken cancellationToken)
        {
            using (var scope = serviceScopeFactory.CreateScope())
            {
                var raceDB       = scope.ServiceProvider.GetService <RaceDB.Models.RaceDBContext>();
                var categoryKeys = raceDB.Category.Where(x => x.Status == 1).ToList();


                foreach (var key in categoryKeys)
                {
                    Func <XPathHandler, XPathHandler> categoryFilter = element => element.Find(TagEnum.Div, "@class='eventWrapper cc_34_8'");
                    var page = await _engine.newPage();

                    await page.GoToAsync(_settings.Bet365.Url.MainPage.ToString());

                    var waitOption = new WaitForSelectorOptions
                    {
                        Timeout = 30000,
                        Hidden  = true
                    };
                    var preLoadOuter = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.PreLoader, waitOption);

                    waitOption.Hidden = false;
                    var selectSport = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.Soccer, waitOption);

                    Thread.Sleep(3000);
                    await selectSport.ClickAsync();

                    Thread.Sleep(3000);
                    var totalLeagueHtml = await page.GetContentAsync();

                    var html       = HtmlHandler.GetImplement("TotalLeague", totalLeagueHtml).Gets(categoryFilter)[0];
                    var attrs      = HtmlHandler.GetImplement("TotalLeague", html).GetsAttributes(_totalLeagueFilter);
                    var values     = HtmlHandler.GetImplement("TotalLeague", html).Gets(_totalLeagueValueFilter);
                    var leagueData = (from a in attrs
                                      from b in values
                                      where a.Key == b.Key
                                      select new
                    {
                        LeagueKey = a.Value.Where(x => x.Key == "data-sportskey").FirstOrDefault().Value,
                        LeagueValue = b.Value
                    });
                    var differentDatas = leagueData.Where(x => raceDB.League.Any(g => g.LeagueName != x.LeagueValue && g.LeagueKey != x.LeagueKey)).ToList();
                    foreach (var data in differentDatas)
                    {
                        var league = raceDB.League.Where(x => x.LeagueName == data.LeagueValue).FirstOrDefault();
                        if (league != null)
                        {
                            league.LeagueKey = data.LeagueKey;
                        }
                        else
                        {
                            RaceDB.Models.League leagueModel = new RaceDB.Models.League();
                            leagueModel.CategoryId = key.CategoryId;
                            leagueModel.LeagueKey  = data.LeagueKey;
                            leagueModel.LeagueName = data.LeagueValue;
                            raceDB.Add(leagueModel);
                        }
                        //.LeagueKey = data.LeagueKey;
                    }
                    await raceDB.SaveChangesAsync();
                }
            }
            return(null);
        }