public async Task CheckRecaptcha(IWebDriver driver, CancellationToken cts)
{
bool captchaCompleted = false;
while (!captchaCompleted)
{
try
{
cts.ThrowIfCancellationRequested();
//var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(15));
//wait.Until(ExpectedConditions.ElementExists(By.ClassName("g-recaptcha")));
var recaptcha = driver.FindElement(By.ClassName("g-recaptcha"));
var responseField = driver.FindElement(By.Id("g-recaptcha-response"));
var sitekey = recaptcha.GetAttribute("data-sitekey");
var values = new Dictionary <string, string>
{
{ "key", _apiKey },
{ "method", "userrecaptcha" },
{ "googlekey", sitekey },
{ "pageurl", driver.Url }
};
var content = new FormUrlEncodedContent(values);
var postResultId = await ScrapeRequest.PostRequest(client, content, cts);
postResultId = postResultId.Substring(3);
var token = await ScrapeRequest.GetRequest(_apiKey, client, postResultId, cts);
token = token.Substring(3);
try
{
var cap = driver.FindElement(By.Id("g-recaptcha-response"));
IJavaScriptExecutor js = (IJavaScriptExecutor)driver;
js.ExecuteScript("document.getElementById('g-recaptcha-response').setAttribute('style', 'display:block;')");
responseField.SendKeys(token);
js.ExecuteScript("handleCaptcha(arguments[0])", token);
break;
} catch (NoSuchElementException)
{
continue;
}
}
catch (NoSuchElementException)
{
captchaCompleted = true;
}
catch (StaleElementReferenceException)
{
captchaCompleted = true;
}
catch (OperationCanceledException)
{
captchaCompleted = true;
}
}
}
public async Task ParseWeb(CancellationToken cts)
{
try
{
Started = true;
_timer.Start();
if (driver == null)
{
var chromeDriverService = ChromeDriverService.CreateDefaultService();
chromeDriverService.HideCommandPromptWindow = true;
var timespan = TimeSpan.FromMinutes(5);
var options = new ChromeOptions();
if (!_showNodes)
{
options.AddArgument("headless");
}
options.AddArguments("--incognito");
driver = new ChromeDriver(chromeDriverService, options, timespan);
driver.Url = RequestUrl;
System.Net.ServicePointManager.Expect100Continue = false;
}
bool indicator = false;
while (!Finished)
{
cts.ThrowIfCancellationRequested();
try
{
var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(1));
wait.Until(ExpectedConditions.ElementExists(By.ClassName("Common-Results-SpinnerWithProgressBar")));
var searchCompleted = driver.FindElement(By.ClassName("Common-Results-SpinnerWithProgressBar"));
var searchCompletedText = searchCompleted.Text;
var results = searchCompleted.FindElement(By.ClassName("count")).Text;
if (searchCompletedText.Contains("Search complete") || searchCompletedText.Contains("Sökningen är klar") || searchCompletedText.Contains("Søket er fullført") || searchCompletedText.Contains("Haku valmis") || searchCompletedText.Contains("Поиск завершен") || searchCompletedText.Contains("Zoektocht compleet") || searchCompletedText.Contains("Suche beendet") || searchCompletedText.Contains("Søgning afsluttet") || searchCompletedText.Contains("Búsqueda finalizada") || searchCompletedText.Contains("Recherche terminée"))
{
if (results.Contains("0"))
{
indicator = true;
driver.Quit();
Finished = true;
break;
}
indicator = true;
cts.ThrowIfCancellationRequested();
var bookingValues = driver.FindElements(By.ClassName("resultWrapper")).ToList();
ScrapeRequest.FileWriter(driver, this, bookingValues);
Finished = true;
driver.Quit();
break;
}
cts.ThrowIfCancellationRequested();
}
catch (WebDriverTimeoutException)
{
try
{
var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(1));
wait.Until(ExpectedConditions.ElementExists(By.ClassName("g-recaptcha")));
await this.CheckRecaptcha(driver, cts);
}
catch (WebDriverTimeoutException)
{
try
{
var prediction = driver.FindElement(By.ClassName("predictionContainer"));
indicator = true;
var bookingValues = driver.FindElements(By.ClassName("resultWrapper")).ToList();
ScrapeRequest.FileWriter(driver, this, bookingValues);
driver.Quit();
Finished = true;
break;
}
catch (NoSuchElementException)
{
continue;
}
catch (Exception)
{
continue;
}
//throw new Exception("LAABAI BLOGAI");
}
}
catch (StaleElementReferenceException)
{
continue;
}
catch (NoSuchElementException)
{
continue;
}
}
_form.Counter -= 1;
_form.progressas.Invoke((System.Windows.Forms.MethodInvoker) delegate
{
{
_form.progressas.Increment(1);
var reiksme = ((double)((double)_form.progressas.Value / (double)_form.progressas.Maximum) * 100).ToString("0.00") + "%";
_form.perc.Text = reiksme;
_timer.Stop();
double est = (double)((double)_counter * (double)_form._bunch.Count) / 60;
_form.es.Text = est.ToString() + " minutes remaining";
}
});
if (_form._bunch.Count != 0)
{
_form._bunch.Pop().ParseWeb(cts);
}
}
catch (OperationCanceledException)
{
_form.progressas.Invoke((MethodInvoker) delegate
{
_form.progressas.Value = 0;
_form.perc.Text = "0";
});
driver.Close();
this.driver.Quit();
}
}
public static void FileWriter(IWebDriver driver, ScrapeRequest request, List <IWebElement> foundResults)
{
List <string> providers = new List <string>();
List <string> prices = new List <string>();
KeyValuePair <string, int> finalValue;
foreach (var value in foundResults)
{
var providerGroup = value.FindElements(By.ClassName("col-price"))
.ToList()
.Select(x => x.FindElement(By.ClassName("above-button")))
.ToList()
.Select(y => y.FindElement(By.ClassName("providerName")))
.ToList()
.Select(z => z.Text)
.ToList();
var priceGroup = value.FindElements(By.ClassName("col-price"))
.ToList()
.Select(x => x.FindElement(By.ClassName("above-button")))
.ToList()
.Select(y => y.FindElement(By.ClassName("price")))
.ToList()
.Select(z => z.Text)
.ToList();
providers.AddRange(providerGroup);
prices.AddRange(priceGroup);
}
List <int> convertedPrices = new List <int>();
foreach (var price in prices)
{
var trimmed = price.Substring(0, price.IndexOf(" "));
if (trimmed.Contains("."))
{
trimmed = trimmed.Remove(trimmed.IndexOf('.'), 1);
}
if (trimmed.Contains(","))
{
trimmed = trimmed.Remove(trimmed.IndexOf(','), 1);
}
var newPrice = Int32.Parse(trimmed);
convertedPrices.Add(newPrice);
}
try
{
List <KeyValuePair <string, int> > values = new List <KeyValuePair <string, int> >();
foreach (var provider in providers)
{
values.Add(new KeyValuePair <string, int>(provider, convertedPrices[providers.IndexOf(provider)]));
}
finalValue = values.OrderBy(x => x.Value).First();
driver.Close();
string location = String.Empty;
if (_form.SaveLocation == null)
{
location = "results.txt";
}
else
{
location = @_form.SaveLocation;
}
using (var stream = new StreamWriter(location, true))
{
if (finalValue.Value <= request._maxPrice)
{
if (request._flightReturn)
{
stream.Write("\\\\\\\\\\ " + "#" + request.Id + " " + request._departureDate + "-" + request._arrivalDate + " \\\\\\\\\\" + Environment.NewLine);
}
else
{
stream.Write("\\\\\\\\\\ " + "#" + request.Id + " " + request._departureDate + "\\\\\\\\\\" + Environment.NewLine);
}
stream.Write("\\\\\\\\\\ Skrydis iš: " + request._departureAirport + " į: " + request._arrivalAirport + " \\\\\\\\\\\\" + Environment.NewLine);
stream.Write("Kompanija: " + finalValue.Key + Environment.NewLine + "Kaina: " + finalValue.Value + Environment.NewLine);
if (request._flightReturn)
{
stream.Write("\\\\\\\\\\ " + request._departureDate + "-" + request._arrivalDate + " \\\\\\\\\\ Pabaiga" + Environment.NewLine + Environment.NewLine);
}
else
{
stream.Write("\\\\\\\\\\ " + request._departureDate + " \\\\\\\\\\ Pabaiga" + Environment.NewLine + Environment.NewLine);
}
}
}
} catch (InvalidOperationException)
{
request.Finished = true;
}
request.Finished = true;
}
private async void scanButton_Click(object sender, EventArgs e)
{
await apiCheck();
if (isValidAPI)
{
cts = new CancellationTokenSource();
progressas = progressBar1;
perc = percentLabel;
es = estimatedLabel;
progressBar1.Value = 0;
if (withReturnRadio.Checked)
{
FlyReturn = true;
}
else
{
FlyReturn = false;
}
if (showNodesCheckBox.Checked)
{
ShowNodes = true;
}
else
{
ShowNodes = false;
}
int tripDurations;
if (FlyReturn)
{
tripDurations = Int32.Parse(tripDurationRangeEnd.Text) - Int32.Parse(tripDurationRangeStart.Text) + 1;
}
else
{
tripDurations = 1;
}
var departureDatesAmmount = (departureDateIntervalEnd.Value - departureDateIntervalStart.Value).Days + 2;
timer = new System.Windows.Forms.Timer();
int duration = 0;
timer.Interval = 1000;
timer.Tick += (s, o) =>
{
duration++;
secondsLabel.Text = duration.ToString();
};
timer.Start();
if (FlyReturn)
{
progressBar1.Maximum = departureDatesAmmount * tripDurations;
}
else
{
progressBar1.Maximum = departureDatesAmmount;
}
var answer = Task.Run(() =>
{
Stack <ScrapeRequest> requests = new Stack <ScrapeRequest>();
int a = 0;
if (FlyReturn)
{
for (int i = 0; i < departureDatesAmmount; i++)
{
for (var j = Int32.Parse(tripDurationRangeStart.Text); j <= Int32.Parse(tripDurationRangeEnd.Text); j++)
{
ScrapeRequest request = new ScrapeRequest(apiInput.Text, ShowNodes, FlyReturn, this, departureAirport.Text, arrivalAirport.Text, departureDateIntervalStart.Value.AddDays(i).ToString("yyyy-MM-dd"), departureDateIntervalStart.Value.AddDays(i).AddDays(j).ToString("yyyy-MM-dd"), domainValue.Text, flightDuration.Text, ++a, Int32.Parse(maxPrice.Text));
requests.Push(request);
}
}
}
else
{
for (int i = 0; i < departureDatesAmmount; i++)
{
ScrapeRequest request = new ScrapeRequest(apiInput.Text, ShowNodes, FlyReturn, this, departureAirport.Text, arrivalAirport.Text, departureDateIntervalStart.Value.AddDays(i).ToString("yyyy-MM-dd"), null, domainValue.Text, flightDuration.Text, ++a, Int32.Parse(maxPrice.Text));
requests.Push(request);
}
}
return(requests);
});
var check = await answer;
_bunch = check;
Counter = Int32.Parse(nodesComboBox.SelectedValue.ToString());
if (answer.Result.Count < Counter)
{
Counter = answer.Result.Count;
}
int inversible = Counter;
List <Task> tasks1 = new List <Task>();
for (int i = 0; i < Counter; i++)
{
new Thread(() =>
{
check.Pop().ParseWeb(cts.Token);
}).Start();
}
}
else
{
MessageBox.Show("Please enter Correct 2captcha.com API key or check balance");
}
//timer.Stop();
//MessageBox.Show("Totally " + duration + " seconds passed");
//duration = 0;
}
