private void Timer_Elapsed(object sender, ElapsedEventArgs e)
{
Log.Information("timer elapsed");
var context = new MariaContext(); //reinstaniating like this means its thread safe
var status = context.NextProfile();
Log.Information("NextProfile finished successfully");
if (status == null)
{
context.Dispose();
return;
}
if (status.Id < End)
{
isWorking = true;
HtmlWeb web = new HtmlWeb();
var doc = web.Load(MakeUrl(status.Id));
Log.Information("Loaded page successfully");
var result = this.Parse(status.Id, doc, status);
Log.Information("parsed successfully");
if (result != null)
{
status.Status = core.models.ProfileStatus.Complete;
context.SetStatusForId(status);
context.Users.Add(result);
context.SaveChanges();
}
else
{
status.Status = core.models.ProfileStatus.ProfileNotPresent;
context.SetStatusForId(status);
}
}
else
{
Log.Information("oh no, in a bad place! <- could be the root cause!");
timer.Stop();
isRunning = false;
}
context.Dispose();
}
public void ParsePost(HtmlNode node, HtmlDocument doc, int postNum, AnnTaskModel task)
{
if (node.InnerText.Contains("They may be unsafe, untrustworthy, or illegal in your jurisdiction.")) //ITS AN AD!
{
return;
}
var model = new PostModel(task);
model.PostNumber = postNum;
model.TopicTitle = model.TopicTitle.RemoveEmojis();
var td = node.Element("td");
if (td == null)
{
throw new Exception("Could not find any tds as children");
}
if (!td.HasChildNodes)
{
return;
}
var table = td.Element("table");
if (table.ChildNodes.Any(f => f.Name == "tbody"))
{
table = table.Element("tbody");
}
table = table.Element("tr").Element("td").Element("table");
if (table.ChildNodes.Any(f => f.Name == "tbody"))
{
table = table.Element("tbody");
}
var details = table.Element("tr");
GetDetails(details, model);
GetPost(details, model);
model.IsScamHeaderPresent = IsPossibleScam(doc);
var context = new MariaContext();
context.Posts.Add(model);
context.SaveChanges();
context.Dispose();
}
