private void _openMainPage(SelenoidStateModel selenoidState, ScraperWinWinStateModel state)
{
var url = $"https://www.winwin.co.il/RealEstate/ForRent/Search/SearchResults/RealEstatePage.aspx?search=8bdb5277c594afddcf9414e7541fd518";
var hasError = false;
var tryCountMax = 20;
var indexTry = 0;
do
{
try
{
hasError = false;
selenoidState.WindowMain.Navigate().GoToUrl(url);
selenoidState.WaitMain.Until(ExpectedConditions.ElementIsVisible(By.Id("carAreasDV")));
}
catch
{
indexTry++;
if (indexTry < tryCountMax)
{
hasError = true;
_initSelenoidBase(selenoidState, state);
Thread.Sleep(1000 * 5);
}
}
} while (hasError);
}
private IReadOnlyCollection <IWebElement> _getRegions(SelenoidStateModel selenoidState)
{
_openRegionClick(selenoidState);
var listOptions = selenoidState.WindowMain.FindElementById("searchDataAreasMD");
var listGoodOptions = listOptions.FindElements(By.ClassName("cSelD"));
return(listGoodOptions);
}
private void _openRegionClick(SelenoidStateModel selenoidState)
{
if (!_isOpenedRegionList(selenoidState))
{
var selectRegion = selenoidState.WindowMain.FindElementById("carAreasDV");
selectRegion.Click();
}
}
private bool _isOpenedRegionList(SelenoidStateModel selenoidState)
{
var selectRegion = selenoidState.WindowMain.FindElementById("box27");
var classes = selectRegion.GetAttribute("class");
var result = classes.Contains("boxTree2On");
return(result);
}
private void _initSelenoid(ScraperHomeLessStateModel state)
{
if (_selenoidState is null)
{
_selenoidState = new SelenoidStateModel();
}
_initSelenoidBase(_selenoidState, state);
}
private void _closeSelenoidBase(SelenoidStateModel selenoidState, ScraperWinWinStateModel state)
{
if (selenoidState.WindowMain != null)
{
selenoidState.WindowMain.Quit();
//_windowMain.Close();
_log($"Close Selenoid Service done");
}
}
private void _scrapeRegion(RegionModel regionModel, SelenoidStateModel selenoidState)
{
var pagination = selenoidState.WindowMain.FindElementByClassName("pagination").FindElements(By.TagName("a"));
var firstPage = pagination.FirstOrDefault();
var lastPageElement = pagination.Skip(pagination.Count() - 2).FirstOrDefault();
var href = firstPage.GetAttribute("href");
var search = href.Split("search=")[1];
var lastPage = lastPageElement.Text;
//var firstNode = region.FindElement(By.TagName("div"));
regionModel.Search = search;
regionModel.AmountPages = Int32.Parse(lastPage);
}
protected bool Selenoid_GoToUrl_Base(string url, SelenoidStateModel selenoidState, IState state)
{
var result = false;
try
{
selenoidState.WindowMain.Navigate().GoToUrl(url);
result = true;
}
catch (Exception exception)
{
_logBase($"Error ff1. {exception.Message}", state);
}
return(result);
}
private void _initSelenoid(ScraperAirdnaStateModel state)
{
if (_selenoidState is null)
{
_selenoidState = new SelenoidStateModel()
{
JavaScriptEnable = true,
ShowPictures = true,
}
}
;
_initSelenoidBase(_selenoidState, state);
}
#endregion
}
private RegionModel _selectRegion(int regionIndex, SelenoidStateModel selenoidState)
{
var result = new RegionModel();
var regions = _getRegions(selenoidState);
_clearRegionsClick(selenoidState);
var region = regions.Skip(regionIndex).FirstOrDefault();
region.Click();
var firstNode = region.FindElement(By.TagName("div"));
result.Id = firstNode.GetAttribute("id");
result.NameRegion = firstNode.GetAttribute("text");
Thread.Sleep(1000 * 2);
_clickSearchButtonAndWait(selenoidState);
return(result);
}
private void _clearRegionsClick(SelenoidStateModel selenoidState)
{
var clearRegions = selenoidState.WindowMain.FindElementByClassName("CancelAllSelections");
clearRegions.Click();
}
private void _clickSearchButton(SelenoidStateModel selenoidState)
{
var searchButton = selenoidState.WindowMain.FindElementById("searchBtnDiv");
searchButton.Click();
}
private void _clickSearchButtonAndWait(SelenoidStateModel selenoidState)
{
_clickSearchButton(selenoidState);
selenoidState.WaitMain.Until(ExpectedConditions.ElementIsVisible(By.Id("carAreasDV")));
}
private async Task _scrapePhase1InnerAsync(ScraperWinWinStateModel state)
{
var hasError = false;
var selenoidState = new SelenoidStateModel()
{
JavaScriptEnable = true,
ShowPictures = false,
};
var indexRegion = 0;
var listRegions = await _loadListRegionsAsync(state);
if (listRegions.Count == 0 || state.IsNew)
{
_initSelenoidBase(selenoidState, state);
_openMainPage(selenoidState, state);
var regions = _getRegions(selenoidState);
foreach (var regionIndex in Enumerable.Range(0, regions.Count))
{
var indexTry = 0;
var tryMax = 50;
do
{
try
{
hasError = false;
var regionModel = _selectRegion(regionIndex, selenoidState);
_scrapeRegion(regionModel, selenoidState);
listRegions.Add(regionModel);
_log($"Scraped region {regionModel.Id}, amount pages: {regionModel.AmountPages}");
indexRegion++;
if (indexRegion % 10 == 0)
{
_saveListRegions(listRegions, state);
}
}
catch (NoSuchElementException)
{
_openMainPage(selenoidState, state);
}
catch (Exception exception)
{
var type = exception.GetType();
indexTry++;
if (indexTry < tryMax)
{
hasError = true;
_initSelenoidBase(selenoidState, state);
Thread.Sleep(1000 * 5);
}
}
} while (hasError);
}
_closeSelenoidBase(selenoidState, state);
_saveListRegions(listRegions, state);
}
}
protected void _initSelenoidBase(SelenoidStateModel selenoidState, IState state)
{
if (selenoidState is null)
{
throw new Exception("Selenoidstate is null");
}
var options = new FirefoxOptions();
if (!selenoidState.ShowPictures)
{
options.SetPreference("permissions.default.image", 2);
}
options.SetPreference("javascript.enabled", selenoidState.JavaScriptEnable);
bool doNeedRepeatRequest = false;
int countRepeat = 0;
int countRepeatMax = 10;
do
{
var selenoidService = _selectSelenoidServiceBase(state);
var selenoidUrl = $"{selenoidService.Protocol}://{selenoidService.Address}:4444/wd/hub";
_logBase($"\tInit Selenoid Service-{state.UsedSelenoidService}:{selenoidUrl}", state);
try
{
if (selenoidState.WindowMain != null)
{
_logBase($"SelenoidService isnt null", state);
selenoidState.WindowMain.Quit();
_logBase($"Quit SelenoidService done", state);
}
_printSelenoidServiceParams(selenoidService, state);
doNeedRepeatRequest = false;
selenoidState.WindowMain = new RemoteWebDriver(new Uri(selenoidUrl), options.ToCapabilities(), TimeSpan.FromMinutes(5));
selenoidState.WaitMain = new WebDriverWait(selenoidState.WindowMain, TimeSpan.FromSeconds(120));
}
catch (Exception exception)
{
_logBase($"Exception e1. {exception.Message}", state);
countRepeat++;
if (countRepeat < countRepeatMax)
{
doNeedRepeatRequest = true;
_logBase($"\t!!! Need change Selenoid Service !!!", state);
_logBase($"\tBad config Selenoid Service-{state.UsedSelenoidService}:", state);
_printSelenoidServiceParams(selenoidService, state);
}
else
{
_logBase($"Try is out. Error", state);
}
}
if (doNeedRepeatRequest)
{
_logBase($"\tcountRepeat:{countRepeat}", state);
state.UsedSelenoidService++;
Thread.Sleep(TimeSpan.FromMinutes(10));
}
} while (doNeedRepeatRequest);
}
