static void Main(string[] args) { Console.WriteLine("Started."); if (IsProduction) { UserDataDir = "/usr/bin/"; IsHeadless = true; AdminPanelApi = "http://127.0.0.1/api/Powertofly"; } else { UserDataDir = "./ChromeDriver/"; IsHeadless = false; AdminPanelApi = "http://localhost:54379/api/Powertofly"; } AdminPanelApi adminPanelApi = new AdminPanelApi(AdminPanelApi); PowerToFlyAPI powerToFlyAPI = new PowerToFlyAPI(); BotSignalDto botSignal = null; while (true) { try { botSignal = adminPanelApi.GetBotSignal(); if (botSignal != null) { adminPanelApi.AddLog($"{DateTime.Now}: Found signal... {botSignal.JobLinks} Start..."); Console.WriteLine("Found signal"); #region LoginUsingSelenium SeleniumWebDriver seleniumWebDriver = null; try { adminPanelApi.AddLog($"{DateTime.Now}: Start login..."); seleniumWebDriver = new SeleniumWebDriver(new SeleniumSettings { IsDisableExtensions = true, IsHeadless = IsHeadless, ProfileDirectory = "Default", UserDataDir = UserDataDir, Port = 39969 }); _ = seleniumWebDriver.ChromeDriver.Manage().Timeouts().ImplicitWait; seleniumWebDriver.ChromeDriver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(20); // Clear cashe seleniumWebDriver.GoToUrl("https://powertofly.com/accounts/login"); Thread.Sleep(5000); seleniumWebDriver.ChromeDriver.Manage().Cookies.DeleteAllCookies(); seleniumWebDriver.GoToUrl("https://powertofly.com/accounts/login"); seleniumWebDriver.ChromeDriver.FindElementById("login-email").SendKeys(botSignal.Email); seleniumWebDriver.ChromeDriver.FindElementById("login-password").SendKeys(botSignal.Password); seleniumWebDriver.ChromeDriver.FindElementById("login-button").Click(); Thread.Sleep(5000); var cookies = seleniumWebDriver.ChromeDriver.Manage().Cookies.AllCookies.ToDictionary(cookie => cookie.Name, cookie => cookie.Value); var session = cookies["session"]; var doc = new HtmlDocument(); doc.LoadHtml(seleniumWebDriver.ChromeDriver.PageSource); var attributes = doc.GetElementbyId("csrf_token").Attributes; var token = attributes.First(x => x.Name == "value").Value; Console.WriteLine("Token: " + token); powerToFlyAPI.SetSession(session); powerToFlyAPI.SetToken(token); seleniumWebDriver.ChromeDriver.Close(); adminPanelApi.AddLog($"{DateTime.Now}: Login successful."); } catch (Exception e) { Console.WriteLine("Catch exception from selenium....."); Console.WriteLine(e.Message); adminPanelApi.AddLog($"{DateTime.Now}: Error during login. Return signal in WAITING status"); adminPanelApi.ChangeSignalStatus(botSignal.Id, BotSignalStatus.Waiting); if (seleniumWebDriver != null) { seleniumWebDriver.ChromeDriver.Close(); } continue; } #endregion //var links = botSignal.JobLinks.Split(";").Where(x => !string.IsNullOrEmpty(x)).Select(x => x.Trim()); var searchLink = botSignal.JobLinks.Trim(); var pageCount = botSignal.MaxPageCount; // Get from bot signal //powerToFlyAPI.Login(botSignal.Email, botSignal.Password); powerToFlyAPI.IsAuth = true; // Get for (int i = 1; i < pageCount; i++) { ICollection <string> jobsByLink = new List <string>(); try { jobsByLink = powerToFlyAPI.GetJobsByLink(searchLink, i).Select(x => x.FullLink).ToList(); adminPanelApi.AddLog($"{DateTime.Now}: Page number {i}. Found {jobsByLink.Count} jobs"); adminPanelApi.UpdateAllJobCountForBotSignal(botSignal.Id, jobsByLink.Count); Thread.Sleep(1000); } catch (Exception e) { adminPanelApi.AddLog($"{DateTime.Now}: Error when parse jobs: Page number {i} not found."); break; } string logTmp = ""; foreach (var item in jobsByLink) { try { var job = powerToFlyAPI.GetJobByLink(item); if (job == null) { logTmp += $"{DateTime.Now}: Job: {item} response already registered. Skip..." + "\n"; continue; } Console.WriteLine("Job find(WithUploadCV): " + job.WithUploadCV + " Redirect: " + job.WithRedirect); logTmp += $"{DateTime.Now}: Job: {item} Can left response: {job.WithUploadCV}. With redirect: {job.WithRedirect}" + "\n"; Thread.Sleep(1000); if (!job.WithRedirect && job.WithUploadCV) { powerToFlyAPI.ApplyForJob(job.ShortLink, botSignal.CoverLetter); adminPanelApi.SaveJob(new JobDto { Name = job.Name, CoverLetter = botSignal.CoverLetter, Link = job.FullLink, SignalId = botSignal.Id }); logTmp += $"{DateTime.Now}: Job: {item} Response left. Saved to DB" + "\n"; } Thread.Sleep(1000); } catch (Exception e) { logTmp += $"{DateTime.Now}: Job: {item} Error during parsing" + "\n"; Console.WriteLine(e.Message); Thread.Sleep(1000); } } adminPanelApi.AddLog(logTmp); } adminPanelApi.ChangeStatus(botSignal.Email, "1"); adminPanelApi.ChangeSignalStatus(botSignal.Id, BotSignalStatus.Finished); botSignal = null; adminPanelApi.AddLog($"{DateTime.Now}: Bot signal finished. Account returned to Available status"); } } catch (Exception e) { Console.WriteLine("Global exception"); Console.WriteLine(e.Message); adminPanelApi.AddLog($"{DateTime.Now}: Bot signal not found"); Thread.Sleep(5000); try { if (botSignal != null) { adminPanelApi.ChangeSignalStatus(botSignal.Id, BotSignalStatus.Waiting); adminPanelApi.ChangeStatus(botSignal.Email, "1"); botSignal = null; } } catch (Exception) { } } } }
private static void TestGetCookies() { //#region Get csrf token using HttpClient //var doc = new HtmlDocument(); //var loginString = Program.client.GetStringAsync("https://powertofly.com/accounts/login").GetAwaiter().GetResult(); //doc.LoadHtml(loginString); //var attributes = doc.GetElementbyId("login-csrf_token").Attributes; //var token = attributes.First(x => x.Name == "value").Value; //#endregion PowerToFlyAPI powerToFlyAPI = new PowerToFlyAPI(); //powerToFlyAPI.Login(); //var job = powerToFlyAPI.GetJobByLink("https://powertofly.com/jobs/detail/331367"); SeleniumWebDriver seleniumWebDriver = new SeleniumWebDriver(new SeleniumSettings { IsDisableExtensions = true, IsHeadless = false, ProfileDirectory = "./ChromeDriver/Profile 2", UserDataDir = @"./ChromeDriver/" }); seleniumWebDriver.GoToUrl("https://powertofly.com/accounts/login"); //seleniumWebDriver.ChromeDriver.FindElementById("login-email").SendKeys(); //seleniumWebDriver.ChromeDriver.FindElementById("login-password").SendKeys(); seleniumWebDriver.ChromeDriver.FindElementById("login-button").Click(); seleniumWebDriver.ChromeDriver.Manage().Cookies.DeleteAllCookies(); Thread.Sleep(5000); var cookies = seleniumWebDriver.ChromeDriver.Manage().Cookies.AllCookies.ToDictionary(cookie => cookie.Name, cookie => cookie.Value); var session = cookies["session"]; var client = new RestClient("https://powertofly.com"); #region Check request for get first session var request = new RestRequest("/api/v1/events/?filter=(ends_at%3E%222020-12-09T22:11:15.639Z%22%20or%20starts_at%3E%222020-12-09T22:11:15.639Z%22)%20and%20etype==%22Chat%20n%20Learn%22%20and%20published_on!=null&order_by=starts_at:asc&fields=tags,slug&include_default=true&per_page=1"); SetHeaders(request, session); var response = client.Get(request); var content = response.Content; //var session = ""; //Program.coockiesDictionary.Add("session", session); #endregion #region First request to site using RestSharp for get session // client.Authenticator = new HttpBasicAuthenticator(username, password); request = new RestRequest("/accounts/login"); SetHeaders(request, session); response = client.Get(request); content = response.Content; // Raw content as string session = response.Cookies.First(x => x.Name == "session").Value; Program.coockiesDictionary.Add("session", session); #endregion #region GetToken var doc = new HtmlDocument(); doc.LoadHtml(content); var attributes = doc.GetElementbyId("csrf_token").Attributes; var token = attributes.First(x => x.Name == "value").Value; #endregion #region Login request = new RestRequest("/accounts/login"); request.AddParameter("login-csrf_token", token); request.AddParameter("login-next", ""); request.AddParameter("login-role", ""); //request.AddParameter("login-email", ); //request.AddParameter("login-password",); SetHeaders(request, session); response = client.Post(request); content = response.Content; // Raw content as string #endregion #region Success request // client.Authenticator = new HttpBasicAuthenticator(username, password); request = new RestRequest("/accounts/login/success"); request.AddHeader("authority", "powertofly.com"); SetHeaders(request, session); response = client.Get(request); content = response.Content; // Raw content as string #endregion #region Load job //https://powertofly.com/jobs/detail/331367 //https://powertofly.com/jobs/detail/331367 request = new RestRequest("/jobs/detail/331367"); request.AddHeader("authority", "powertofly.com"); SetHeaders(request, session); response = client.Get(request); content = response.Content; // Raw content as string #endregion #region Apply on job request = new RestRequest("/jobs/apply/371282.json?confirm_skills=True"); request.AddHeader("authority", "powertofly.com"); request.AddParameter("csrf_token", token); request.AddParameter("cover_letter", "Sorry I just test how this site work"); request.AddParameter("submit", "Apply"); request.AddParameter("next", "https://powertofly.com/jobs/detail/371282"); SetHeaders(request, session); response = client.Post(request); content = response.Content; // Raw content as string #endregion //client = new RestClient("https://powertofly.com"); //// client.Authenticator = new HttpBasicAuthenticator(username, password); //request = new RestRequest("/accounts/login"); //request.AddParameter("login-csrf_token", token); //request.AddParameter("login-next", ""); //request.AddParameter("login-role", ""); //request.AddParameter("login-email", "*****@*****.**"); //request.AddHeader("header", "value"); //response = client.Post(request); //content = response.Content; // Raw content as string //foreach (var item in response.Cookies) //{ // Console.WriteLine(item); //} //var values = new Dictionary<string, string> //{ // { "login-csrf_token", token }, // { "login-next", "" }, // { "login-role", "" }, //}; //var content = new FormUrlEncodedContent(values); //var response = client.PostAsync("https://powertofly.com/accounts/login", content).GetAwaiter().GetResult(); //var responseString = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); //string uri = "https://powertofly.com/accounts/login"; //HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri); //request.Method = WebRequestMethods.Http.Get; //request.AllowAutoRedirect = false; //request.CookieContainer = new CookieContainer(); //request.KeepAlive = true; //request.ContentType = "application/x-www-form-urlencoded"; //request.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5"; //HttpWebResponse response = (HttpWebResponse)request.GetResponse(); //Stream receiveStream = response.GetResponseStream(); //StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8); //string oku = readStream.ReadToEnd(); //foreach (Cookie cook in response.Cookies) //{ // Console.WriteLine("Domain: {0}, Name: {1}, value: {2}", cook.Domain, cook.Name, cook.Value); //} }