Exemple #1
0
        public async Task CheckRecaptcha(IWebDriver driver, CancellationToken cts)
        {
            bool captchaCompleted = false;

            while (!captchaCompleted)
            {
                try
                {
                    cts.ThrowIfCancellationRequested();
                    //var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(15));
                    //wait.Until(ExpectedConditions.ElementExists(By.ClassName("g-recaptcha")));
                    var recaptcha     = driver.FindElement(By.ClassName("g-recaptcha"));
                    var responseField = driver.FindElement(By.Id("g-recaptcha-response"));
                    var sitekey       = recaptcha.GetAttribute("data-sitekey");
                    var values        = new Dictionary <string, string>
                    {
                        { "key", _apiKey },
                        { "method", "userrecaptcha" },
                        { "googlekey", sitekey },
                        { "pageurl", driver.Url }
                    };
                    var content      = new FormUrlEncodedContent(values);
                    var postResultId = await ScrapeRequest.PostRequest(client, content, cts);

                    postResultId = postResultId.Substring(3);
                    var token = await ScrapeRequest.GetRequest(_apiKey, client, postResultId, cts);

                    token = token.Substring(3);
                    try
                    {
                        var cap = driver.FindElement(By.Id("g-recaptcha-response"));
                        IJavaScriptExecutor js = (IJavaScriptExecutor)driver;
                        js.ExecuteScript("document.getElementById('g-recaptcha-response').setAttribute('style', 'display:block;')");
                        responseField.SendKeys(token);
                        js.ExecuteScript("handleCaptcha(arguments[0])", token);
                        break;
                    } catch (NoSuchElementException)
                    {
                        continue;
                    }
                }
                catch (NoSuchElementException)
                {
                    captchaCompleted = true;
                }
                catch (StaleElementReferenceException)
                {
                    captchaCompleted = true;
                }
                catch (OperationCanceledException)
                {
                    captchaCompleted = true;
                }
            }
        }
Exemple #2
0
        public async Task ParseWeb(CancellationToken cts)
        {
            try
            {
                Started = true;
                _timer.Start();
                if (driver == null)
                {
                    var chromeDriverService = ChromeDriverService.CreateDefaultService();
                    chromeDriverService.HideCommandPromptWindow = true;
                    var timespan = TimeSpan.FromMinutes(5);
                    var options  = new ChromeOptions();
                    if (!_showNodes)
                    {
                        options.AddArgument("headless");
                    }
                    options.AddArguments("--incognito");
                    driver     = new ChromeDriver(chromeDriverService, options, timespan);
                    driver.Url = RequestUrl;
                    System.Net.ServicePointManager.Expect100Continue = false;
                }
                bool indicator = false;

                while (!Finished)
                {
                    cts.ThrowIfCancellationRequested();
                    try
                    {
                        var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(1));
                        wait.Until(ExpectedConditions.ElementExists(By.ClassName("Common-Results-SpinnerWithProgressBar")));

                        var searchCompleted     = driver.FindElement(By.ClassName("Common-Results-SpinnerWithProgressBar"));
                        var searchCompletedText = searchCompleted.Text;
                        var results             = searchCompleted.FindElement(By.ClassName("count")).Text;

                        if (searchCompletedText.Contains("Search complete") || searchCompletedText.Contains("Sökningen är klar") || searchCompletedText.Contains("Søket er fullført") || searchCompletedText.Contains("Haku valmis") || searchCompletedText.Contains("Поиск завершен") || searchCompletedText.Contains("Zoektocht compleet") || searchCompletedText.Contains("Suche beendet") || searchCompletedText.Contains("Søgning afsluttet") || searchCompletedText.Contains("Búsqueda finalizada") || searchCompletedText.Contains("Recherche terminée"))
                        {
                            if (results.Contains("0"))
                            {
                                indicator = true;
                                driver.Quit();
                                Finished = true;
                                break;
                            }
                            indicator = true;

                            cts.ThrowIfCancellationRequested();
                            var bookingValues = driver.FindElements(By.ClassName("resultWrapper")).ToList();

                            ScrapeRequest.FileWriter(driver, this, bookingValues);
                            Finished = true;
                            driver.Quit();
                            break;
                        }
                        cts.ThrowIfCancellationRequested();
                    }
                    catch (WebDriverTimeoutException)
                    {
                        try
                        {
                            var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(1));
                            wait.Until(ExpectedConditions.ElementExists(By.ClassName("g-recaptcha")));
                            await this.CheckRecaptcha(driver, cts);
                        }
                        catch (WebDriverTimeoutException)
                        {
                            try
                            {
                                var prediction = driver.FindElement(By.ClassName("predictionContainer"));
                                indicator = true;
                                var bookingValues = driver.FindElements(By.ClassName("resultWrapper")).ToList();
                                ScrapeRequest.FileWriter(driver, this, bookingValues);
                                driver.Quit();
                                Finished = true;
                                break;
                            }
                            catch (NoSuchElementException)
                            {
                                continue;
                            }
                            catch (Exception)
                            {
                                continue;
                            }
                            //throw new Exception("LAABAI BLOGAI");
                        }
                    }
                    catch (StaleElementReferenceException)
                    {
                        continue;
                    }
                    catch (NoSuchElementException)
                    {
                        continue;
                    }
                }

                _form.Counter -= 1;
                _form.progressas.Invoke((System.Windows.Forms.MethodInvoker) delegate
                {
                    {
                        _form.progressas.Increment(1);
                        var reiksme     = ((double)((double)_form.progressas.Value / (double)_form.progressas.Maximum) * 100).ToString("0.00") + "%";
                        _form.perc.Text = reiksme;
                        _timer.Stop();
                        double est    = (double)((double)_counter * (double)_form._bunch.Count) / 60;
                        _form.es.Text = est.ToString() + " minutes remaining";
                    }
                });
                if (_form._bunch.Count != 0)
                {
                    _form._bunch.Pop().ParseWeb(cts);
                }
            }
            catch (OperationCanceledException)
            {
                _form.progressas.Invoke((MethodInvoker) delegate
                {
                    _form.progressas.Value = 0;
                    _form.perc.Text        = "0";
                });
                driver.Close();
                this.driver.Quit();
            }
        }
Exemple #3
0
        public static void FileWriter(IWebDriver driver, ScrapeRequest request, List <IWebElement> foundResults)
        {
            List <string> providers = new List <string>();
            List <string> prices    = new List <string>();
            KeyValuePair <string, int> finalValue;

            foreach (var value in foundResults)
            {
                var providerGroup = value.FindElements(By.ClassName("col-price"))
                                    .ToList()
                                    .Select(x => x.FindElement(By.ClassName("above-button")))
                                    .ToList()
                                    .Select(y => y.FindElement(By.ClassName("providerName")))
                                    .ToList()
                                    .Select(z => z.Text)
                                    .ToList();
                var priceGroup = value.FindElements(By.ClassName("col-price"))
                                 .ToList()
                                 .Select(x => x.FindElement(By.ClassName("above-button")))
                                 .ToList()
                                 .Select(y => y.FindElement(By.ClassName("price")))
                                 .ToList()
                                 .Select(z => z.Text)
                                 .ToList();

                providers.AddRange(providerGroup);
                prices.AddRange(priceGroup);
            }

            List <int> convertedPrices = new List <int>();

            foreach (var price in prices)
            {
                var trimmed = price.Substring(0, price.IndexOf(" "));
                if (trimmed.Contains("."))
                {
                    trimmed = trimmed.Remove(trimmed.IndexOf('.'), 1);
                }
                if (trimmed.Contains(","))
                {
                    trimmed = trimmed.Remove(trimmed.IndexOf(','), 1);
                }
                var newPrice = Int32.Parse(trimmed);
                convertedPrices.Add(newPrice);
            }

            try
            {
                List <KeyValuePair <string, int> > values = new List <KeyValuePair <string, int> >();
                foreach (var provider in providers)
                {
                    values.Add(new KeyValuePair <string, int>(provider, convertedPrices[providers.IndexOf(provider)]));
                }
                finalValue = values.OrderBy(x => x.Value).First();
                driver.Close();
                string location = String.Empty;
                if (_form.SaveLocation == null)
                {
                    location = "results.txt";
                }
                else
                {
                    location = @_form.SaveLocation;
                }
                using (var stream = new StreamWriter(location, true))
                {
                    if (finalValue.Value <= request._maxPrice)
                    {
                        if (request._flightReturn)
                        {
                            stream.Write("\\\\\\\\\\ " + "#" + request.Id + " " + request._departureDate + "-" + request._arrivalDate + " \\\\\\\\\\" + Environment.NewLine);
                        }
                        else
                        {
                            stream.Write("\\\\\\\\\\ " + "#" + request.Id + " " + request._departureDate + "\\\\\\\\\\" + Environment.NewLine);
                        }
                        stream.Write("\\\\\\\\\\ Skrydis iš: " + request._departureAirport + " į: " + request._arrivalAirport + " \\\\\\\\\\\\" + Environment.NewLine);
                        stream.Write("Kompanija: " + finalValue.Key + Environment.NewLine + "Kaina: " + finalValue.Value + Environment.NewLine);
                        if (request._flightReturn)
                        {
                            stream.Write("\\\\\\\\\\ " + request._departureDate + "-" + request._arrivalDate + " \\\\\\\\\\ Pabaiga" + Environment.NewLine + Environment.NewLine);
                        }
                        else
                        {
                            stream.Write("\\\\\\\\\\ " + request._departureDate + " \\\\\\\\\\ Pabaiga" + Environment.NewLine + Environment.NewLine);
                        }
                    }
                }
            } catch (InvalidOperationException)
            {
                request.Finished = true;
            }
            request.Finished = true;
        }
Exemple #4
0
        private async void scanButton_Click(object sender, EventArgs e)
        {
            await apiCheck();

            if (isValidAPI)
            {
                cts                = new CancellationTokenSource();
                progressas         = progressBar1;
                perc               = percentLabel;
                es                 = estimatedLabel;
                progressBar1.Value = 0;
                if (withReturnRadio.Checked)
                {
                    FlyReturn = true;
                }
                else
                {
                    FlyReturn = false;
                }
                if (showNodesCheckBox.Checked)
                {
                    ShowNodes = true;
                }
                else
                {
                    ShowNodes = false;
                }
                int tripDurations;
                if (FlyReturn)
                {
                    tripDurations = Int32.Parse(tripDurationRangeEnd.Text) - Int32.Parse(tripDurationRangeStart.Text) + 1;
                }
                else
                {
                    tripDurations = 1;
                }
                var departureDatesAmmount = (departureDateIntervalEnd.Value - departureDateIntervalStart.Value).Days + 2;
                timer = new System.Windows.Forms.Timer();
                int duration = 0;
                timer.Interval = 1000;
                timer.Tick    += (s, o) =>
                {
                    duration++;
                    secondsLabel.Text = duration.ToString();
                };
                timer.Start();
                if (FlyReturn)
                {
                    progressBar1.Maximum = departureDatesAmmount * tripDurations;
                }
                else
                {
                    progressBar1.Maximum = departureDatesAmmount;
                }
                var answer = Task.Run(() =>
                {
                    Stack <ScrapeRequest> requests = new Stack <ScrapeRequest>();
                    int a = 0;
                    if (FlyReturn)
                    {
                        for (int i = 0; i < departureDatesAmmount; i++)
                        {
                            for (var j = Int32.Parse(tripDurationRangeStart.Text); j <= Int32.Parse(tripDurationRangeEnd.Text); j++)
                            {
                                ScrapeRequest request = new ScrapeRequest(apiInput.Text, ShowNodes, FlyReturn, this, departureAirport.Text, arrivalAirport.Text, departureDateIntervalStart.Value.AddDays(i).ToString("yyyy-MM-dd"), departureDateIntervalStart.Value.AddDays(i).AddDays(j).ToString("yyyy-MM-dd"), domainValue.Text, flightDuration.Text, ++a, Int32.Parse(maxPrice.Text));
                                requests.Push(request);
                            }
                        }
                    }
                    else
                    {
                        for (int i = 0; i < departureDatesAmmount; i++)
                        {
                            ScrapeRequest request = new ScrapeRequest(apiInput.Text, ShowNodes, FlyReturn, this, departureAirport.Text, arrivalAirport.Text, departureDateIntervalStart.Value.AddDays(i).ToString("yyyy-MM-dd"), null, domainValue.Text, flightDuration.Text, ++a, Int32.Parse(maxPrice.Text));
                            requests.Push(request);
                        }
                    }

                    return(requests);
                });

                var check = await answer;
                _bunch  = check;
                Counter = Int32.Parse(nodesComboBox.SelectedValue.ToString());

                if (answer.Result.Count < Counter)
                {
                    Counter = answer.Result.Count;
                }

                int         inversible = Counter;
                List <Task> tasks1     = new List <Task>();
                for (int i = 0; i < Counter; i++)
                {
                    new Thread(() =>
                    {
                        check.Pop().ParseWeb(cts.Token);
                    }).Start();
                }
            }
            else
            {
                MessageBox.Show("Please enter Correct 2captcha.com API key or check balance");
            }
            //timer.Stop();
            //MessageBox.Show("Totally " + duration + " seconds passed");
            //duration = 0;
        }