static void Main(string[] args)
{
Console.WriteLine("Started.");
if (IsProduction)
{
UserDataDir = "/usr/bin/";
IsHeadless = true;
AdminPanelApi = "http://127.0.0.1/api/Powertofly";
}
else
{
UserDataDir = "./ChromeDriver/";
IsHeadless = false;
AdminPanelApi = "http://localhost:54379/api/Powertofly";
}
AdminPanelApi adminPanelApi = new AdminPanelApi(AdminPanelApi);
PowerToFlyAPI powerToFlyAPI = new PowerToFlyAPI();
BotSignalDto botSignal = null;
while (true)
{
try
{
botSignal = adminPanelApi.GetBotSignal();
if (botSignal != null)
{
adminPanelApi.AddLog($"{DateTime.Now}: Found signal... {botSignal.JobLinks} Start...");
Console.WriteLine("Found signal");
#region LoginUsingSelenium
SeleniumWebDriver seleniumWebDriver = null;
try
{
adminPanelApi.AddLog($"{DateTime.Now}: Start login...");
seleniumWebDriver = new SeleniumWebDriver(new SeleniumSettings
{
IsDisableExtensions = true,
IsHeadless = IsHeadless,
ProfileDirectory = "Default",
UserDataDir = UserDataDir,
Port = 39969
});
_ = seleniumWebDriver.ChromeDriver.Manage().Timeouts().ImplicitWait;
seleniumWebDriver.ChromeDriver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(20);
// Clear cashe
seleniumWebDriver.GoToUrl("https://powertofly.com/accounts/login");
Thread.Sleep(5000);
seleniumWebDriver.ChromeDriver.Manage().Cookies.DeleteAllCookies();
seleniumWebDriver.GoToUrl("https://powertofly.com/accounts/login");
seleniumWebDriver.ChromeDriver.FindElementById("login-email").SendKeys(botSignal.Email);
seleniumWebDriver.ChromeDriver.FindElementById("login-password").SendKeys(botSignal.Password);
seleniumWebDriver.ChromeDriver.FindElementById("login-button").Click();
Thread.Sleep(5000);
var cookies = seleniumWebDriver.ChromeDriver.Manage().Cookies.AllCookies.ToDictionary(cookie => cookie.Name, cookie => cookie.Value);
var session = cookies["session"];
var doc = new HtmlDocument();
doc.LoadHtml(seleniumWebDriver.ChromeDriver.PageSource);
var attributes = doc.GetElementbyId("csrf_token").Attributes;
var token = attributes.First(x => x.Name == "value").Value;
Console.WriteLine("Token: " + token);
powerToFlyAPI.SetSession(session);
powerToFlyAPI.SetToken(token);
seleniumWebDriver.ChromeDriver.Close();
adminPanelApi.AddLog($"{DateTime.Now}: Login successful.");
}
catch (Exception e)
{
Console.WriteLine("Catch exception from selenium.....");
Console.WriteLine(e.Message);
adminPanelApi.AddLog($"{DateTime.Now}: Error during login. Return signal in WAITING status");
adminPanelApi.ChangeSignalStatus(botSignal.Id, BotSignalStatus.Waiting);
if (seleniumWebDriver != null)
{
seleniumWebDriver.ChromeDriver.Close();
}
continue;
}
#endregion
//var links = botSignal.JobLinks.Split(";").Where(x => !string.IsNullOrEmpty(x)).Select(x => x.Trim());
var searchLink = botSignal.JobLinks.Trim();
var pageCount = botSignal.MaxPageCount; // Get from bot signal
//powerToFlyAPI.Login(botSignal.Email, botSignal.Password);
powerToFlyAPI.IsAuth = true;
// Get
for (int i = 1; i < pageCount; i++)
{
ICollection <string> jobsByLink = new List <string>();
try
{
jobsByLink = powerToFlyAPI.GetJobsByLink(searchLink, i).Select(x => x.FullLink).ToList();
adminPanelApi.AddLog($"{DateTime.Now}: Page number {i}. Found {jobsByLink.Count} jobs");
adminPanelApi.UpdateAllJobCountForBotSignal(botSignal.Id, jobsByLink.Count);
Thread.Sleep(1000);
}
catch (Exception e)
{
adminPanelApi.AddLog($"{DateTime.Now}: Error when parse jobs: Page number {i} not found.");
break;
}
string logTmp = "";
foreach (var item in jobsByLink)
{
try
{
var job = powerToFlyAPI.GetJobByLink(item);
if (job == null)
{
logTmp += $"{DateTime.Now}: Job: {item} response already registered. Skip..." + "\n";
continue;
}
Console.WriteLine("Job find(WithUploadCV): " + job.WithUploadCV + " Redirect: " + job.WithRedirect);
logTmp += $"{DateTime.Now}: Job: {item} Can left response: {job.WithUploadCV}. With redirect: {job.WithRedirect}" + "\n";
Thread.Sleep(1000);
if (!job.WithRedirect && job.WithUploadCV)
{
powerToFlyAPI.ApplyForJob(job.ShortLink, botSignal.CoverLetter);
adminPanelApi.SaveJob(new JobDto {
Name = job.Name, CoverLetter = botSignal.CoverLetter, Link = job.FullLink, SignalId = botSignal.Id
});
logTmp += $"{DateTime.Now}: Job: {item} Response left. Saved to DB" + "\n";
}
Thread.Sleep(1000);
}
catch (Exception e)
{
logTmp += $"{DateTime.Now}: Job: {item} Error during parsing" + "\n";
Console.WriteLine(e.Message);
Thread.Sleep(1000);
}
}
adminPanelApi.AddLog(logTmp);
}
adminPanelApi.ChangeStatus(botSignal.Email, "1");
adminPanelApi.ChangeSignalStatus(botSignal.Id, BotSignalStatus.Finished);
botSignal = null;
adminPanelApi.AddLog($"{DateTime.Now}: Bot signal finished. Account returned to Available status");
}
}
catch (Exception e)
{
Console.WriteLine("Global exception");
Console.WriteLine(e.Message);
adminPanelApi.AddLog($"{DateTime.Now}: Bot signal not found");
Thread.Sleep(5000);
try
{
if (botSignal != null)
{
adminPanelApi.ChangeSignalStatus(botSignal.Id, BotSignalStatus.Waiting);
adminPanelApi.ChangeStatus(botSignal.Email, "1");
botSignal = null;
}
}
catch (Exception)
{
}
}
}
}
private static void TestGetCookies()
{
//#region Get csrf token using HttpClient
//var doc = new HtmlDocument();
//var loginString = Program.client.GetStringAsync("https://powertofly.com/accounts/login").GetAwaiter().GetResult();
//doc.LoadHtml(loginString);
//var attributes = doc.GetElementbyId("login-csrf_token").Attributes;
//var token = attributes.First(x => x.Name == "value").Value;
//#endregion
PowerToFlyAPI powerToFlyAPI = new PowerToFlyAPI();
//powerToFlyAPI.Login();
//var job = powerToFlyAPI.GetJobByLink("https://powertofly.com/jobs/detail/331367");
SeleniumWebDriver seleniumWebDriver = new SeleniumWebDriver(new SeleniumSettings
{
IsDisableExtensions = true,
IsHeadless = false,
ProfileDirectory = "./ChromeDriver/Profile 2",
UserDataDir = @"./ChromeDriver/"
});
seleniumWebDriver.GoToUrl("https://powertofly.com/accounts/login");
//seleniumWebDriver.ChromeDriver.FindElementById("login-email").SendKeys();
//seleniumWebDriver.ChromeDriver.FindElementById("login-password").SendKeys();
seleniumWebDriver.ChromeDriver.FindElementById("login-button").Click();
seleniumWebDriver.ChromeDriver.Manage().Cookies.DeleteAllCookies();
Thread.Sleep(5000);
var cookies = seleniumWebDriver.ChromeDriver.Manage().Cookies.AllCookies.ToDictionary(cookie => cookie.Name, cookie => cookie.Value);
var session = cookies["session"];
var client = new RestClient("https://powertofly.com");
#region Check request for get first session
var request = new RestRequest("/api/v1/events/?filter=(ends_at%3E%222020-12-09T22:11:15.639Z%22%20or%20starts_at%3E%222020-12-09T22:11:15.639Z%22)%20and%20etype==%22Chat%20n%20Learn%22%20and%20published_on!=null&order_by=starts_at:asc&fields=tags,slug&include_default=true&per_page=1");
SetHeaders(request, session);
var response = client.Get(request);
var content = response.Content;
//var session = "";
//Program.coockiesDictionary.Add("session", session);
#endregion
#region First request to site using RestSharp for get session
// client.Authenticator = new HttpBasicAuthenticator(username, password);
request = new RestRequest("/accounts/login");
SetHeaders(request, session);
response = client.Get(request);
content = response.Content; // Raw content as string
session = response.Cookies.First(x => x.Name == "session").Value;
Program.coockiesDictionary.Add("session", session);
#endregion
#region GetToken
var doc = new HtmlDocument();
doc.LoadHtml(content);
var attributes = doc.GetElementbyId("csrf_token").Attributes;
var token = attributes.First(x => x.Name == "value").Value;
#endregion
#region Login
request = new RestRequest("/accounts/login");
request.AddParameter("login-csrf_token", token);
request.AddParameter("login-next", "");
request.AddParameter("login-role", "");
//request.AddParameter("login-email", );
//request.AddParameter("login-password",);
SetHeaders(request, session);
response = client.Post(request);
content = response.Content; // Raw content as string
#endregion
#region Success request
// client.Authenticator = new HttpBasicAuthenticator(username, password);
request = new RestRequest("/accounts/login/success");
request.AddHeader("authority", "powertofly.com");
SetHeaders(request, session);
response = client.Get(request);
content = response.Content; // Raw content as string
#endregion
#region Load job
//https://powertofly.com/jobs/detail/331367
//https://powertofly.com/jobs/detail/331367
request = new RestRequest("/jobs/detail/331367");
request.AddHeader("authority", "powertofly.com");
SetHeaders(request, session);
response = client.Get(request);
content = response.Content; // Raw content as string
#endregion
#region Apply on job
request = new RestRequest("/jobs/apply/371282.json?confirm_skills=True");
request.AddHeader("authority", "powertofly.com");
request.AddParameter("csrf_token", token);
request.AddParameter("cover_letter", "Sorry I just test how this site work");
request.AddParameter("submit", "Apply");
request.AddParameter("next", "https://powertofly.com/jobs/detail/371282");
SetHeaders(request, session);
response = client.Post(request);
content = response.Content; // Raw content as string
#endregion
//client = new RestClient("https://powertofly.com");
//// client.Authenticator = new HttpBasicAuthenticator(username, password);
//request = new RestRequest("/accounts/login");
//request.AddParameter("login-csrf_token", token);
//request.AddParameter("login-next", "");
//request.AddParameter("login-role", "");
//request.AddParameter("login-email", "*****@*****.**");
//request.AddHeader("header", "value");
//response = client.Post(request);
//content = response.Content; // Raw content as string
//foreach (var item in response.Cookies)
//{
// Console.WriteLine(item);
//}
//var values = new Dictionary<string, string>
//{
// { "login-csrf_token", token },
// { "login-next", "" },
// { "login-role", "" },
//};
//var content = new FormUrlEncodedContent(values);
//var response = client.PostAsync("https://powertofly.com/accounts/login", content).GetAwaiter().GetResult();
//var responseString = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
//string uri = "https://powertofly.com/accounts/login";
//HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
//request.Method = WebRequestMethods.Http.Get;
//request.AllowAutoRedirect = false;
//request.CookieContainer = new CookieContainer();
//request.KeepAlive = true;
//request.ContentType = "application/x-www-form-urlencoded";
//request.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5";
//HttpWebResponse response = (HttpWebResponse)request.GetResponse();
//Stream receiveStream = response.GetResponseStream();
//StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8);
//string oku = readStream.ReadToEnd();
//foreach (Cookie cook in response.Cookies)
//{
// Console.WriteLine("Domain: {0}, Name: {1}, value: {2}", cook.Domain, cook.Name, cook.Value);
//}
}