private void Timer_Elapsed(object sender, ElapsedEventArgs e) { Log.Information("timer elapsed"); var context = new MariaContext(); //reinstaniating like this means its thread safe var status = context.NextProfile(); Log.Information("NextProfile finished successfully"); if (status == null) { context.Dispose(); return; } if (status.Id < End) { isWorking = true; HtmlWeb web = new HtmlWeb(); var doc = web.Load(MakeUrl(status.Id)); Log.Information("Loaded page successfully"); var result = this.Parse(status.Id, doc, status); Log.Information("parsed successfully"); if (result != null) { status.Status = core.models.ProfileStatus.Complete; context.SetStatusForId(status); context.Users.Add(result); context.SaveChanges(); } else { status.Status = core.models.ProfileStatus.ProfileNotPresent; context.SetStatusForId(status); } } else { Log.Information("oh no, in a bad place! <- could be the root cause!"); timer.Stop(); isRunning = false; } context.Dispose(); }
public void ParsePost(HtmlNode node, HtmlDocument doc, int postNum, AnnTaskModel task) { if (node.InnerText.Contains("They may be unsafe, untrustworthy, or illegal in your jurisdiction.")) //ITS AN AD! { return; } var model = new PostModel(task); model.PostNumber = postNum; model.TopicTitle = model.TopicTitle.RemoveEmojis(); var td = node.Element("td"); if (td == null) { throw new Exception("Could not find any tds as children"); } if (!td.HasChildNodes) { return; } var table = td.Element("table"); if (table.ChildNodes.Any(f => f.Name == "tbody")) { table = table.Element("tbody"); } table = table.Element("tr").Element("td").Element("table"); if (table.ChildNodes.Any(f => f.Name == "tbody")) { table = table.Element("tbody"); } var details = table.Element("tr"); GetDetails(details, model); GetPost(details, model); model.IsScamHeaderPresent = IsPossibleScam(doc); var context = new MariaContext(); context.Posts.Add(model); context.SaveChanges(); context.Dispose(); }