public Task <dynamic> UploadUrl(int start = 1, int end = 0) { int pages = 0; if (end == 0) { pages = GetLastPage(); } else { pages = end; } Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); var enc1252 = Encoding.GetEncoding("Windows-1251"); var htmlWeb = new HtmlWeb(); htmlWeb.OverrideEncoding = enc1252; for (int i = start; i < pages; i++) { var baseUrl = "https://www.weblancer.net/freelancers/"; if (i > 1) { baseUrl += "?page=" + i; } var linkHTML = htmlWeb.Load(baseUrl); var nodes = linkHTML.DocumentNode.SelectNodes("//div[@class='cols_table divided_rows freelancers']//div[@class='row']"); foreach (var node in nodes) { using (var db = new WeblancerDbContext()) { var login = node.SelectSingleNode("//div[@class='user_brief']").Attributes["data-login"].Value; var userUrl = "https://www.weblancer.net/users/" + login + "/"; var urlExist = db.DevtoCandidates.Where(c => c.Url == userUrl).FirstOrDefault(); var urlData = new DevtoCandidates(); urlData.Url = userUrl; if (urlExist == null) { db.DevtoCandidates.Add(urlData); db.SaveChanges(); Console.WriteLine(urlData.Url); } } } } return(null); }
public void DataLoader(int start = 0, int length = 0) { Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); var enc1252 = Encoding.GetEncoding("Windows-1251"); var htmlWeb = new HtmlWeb(); List <DevtoCandidates> devs = null; using (var db = new WeblancerDbContext()) { if (start == 0 && length == 0) { devs = db.DevtoCandidates.AsNoTracking().ToList(); } else { devs = db.DevtoCandidates.AsNoTracking().Where(v => v.Id >= start).Take(length).ToList(); } } foreach (var c in devs) { var linkHTML = htmlWeb.Load(c.Url); var login = linkHTML.DocumentNode.SelectSingleNode("//div[@class='user_brief']").Attributes["data-login"].Value; var breif = linkHTML.DocumentNode.SelectSingleNode("//div[@class='user_brief']//div[@class='brief']").InnerText; var weblancer = new WeblancerCandidates(); weblancer.Url_Id = c.Id; weblancer.Login = login; weblancer.Cont_Content = breif; List <WeblancerCandidates> workerExist = null; using (var db = new WeblancerDbContext()) { workerExist = db.WeblancerCandidates.Where(w => w.Url_Id == c.Id).ToList(); } using (var db = new WeblancerDbContext()) { if (workerExist.Count == 0) { db.WeblancerCandidates.Add(weblancer); Console.WriteLine("Successfully added: " + weblancer.Login); } else { db.WeblancerCandidates.Update(weblancer); Console.WriteLine("Successfully updated: " + weblancer.Login); }; db.SaveChanges(); } } }