private void DoScrape(string siteUsername, string sitePassword, string source)
{
var dbContextFactory = new DesignTimeDbContextFactory();
using (var db = dbContextFactory.CreateDbContext(connectionString, Log.Logger))
{
// Note!
// the user needs to be added before we disable tracking and disable AutoDetectChanges
// otherwise this will crash
// set admin user
var adminUser = new User()
{
FirstName = "",
LastName = "Admin",
UserName = "******"
};
// check if user already exists
var existingUser = db.DictionaryUsers.Where(u => u.FirstName == adminUser.FirstName).FirstOrDefault();
if (existingUser != null)
{
adminUser = existingUser;
}
else
{
db.DictionaryUsers.Add(adminUser);
db.SaveChanges();
}
// disable tracking to speed things up
// note that this doesn't load the virtual properties, but loads the object ids after a save
db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
// this doesn't seem to work when adding new users all the time
db.ChangeTracker.AutoDetectChangesEnabled = false;
using (var driver = ChromeDriverUtils.GetChromeDriver(true))
{
DoLogon(driver, siteUsername, sitePassword);
string url = "https://www.kryssord.org";
driver.Navigate().GoToUrl(url);
var documentNode = driver.GetDocumentNode();
ProcessWordsUntilEmpty(driver, db, adminUser, documentNode);
}
}
}
private void DoScrape(int startLetterCount, int endLetterCount, string source, bool doContinueWithLastWord)
{
var dbContextFactory = new DesignTimeDbContextFactory();
using (var db = dbContextFactory.CreateDbContext(connectionString, Log.Logger))
{
// Note!
// the user needs to be added before we disable tracking and disable AutoDetectChanges
// otherwise this will crash
// set admin user
var adminUser = new User()
{
FirstName = "",
LastName = "Admin",
UserName = "******"
};
// check if user already exists
var existingUser = db.DictionaryUsers.Where(u => u.FirstName == adminUser.FirstName).FirstOrDefault();
if (existingUser != null)
{
adminUser = existingUser;
}
else
{
db.DictionaryUsers.Add(adminUser);
db.SaveChanges();
}
// disable tracking to speed things up
// note that this doesn't load the virtual properties, but loads the object ids after a save
db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
// this doesn't seem to work when adding new users all the time
db.ChangeTracker.AutoDetectChangesEnabled = false;
using (var driver = ChromeDriverUtils.GetChromeDriver(true))
{
// set general timeout to long
driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(180);
// read all words with the letter count
ReadWordsByWordPermutations(startLetterCount, endLetterCount, driver, db, adminUser, doContinueWithLastWord);
}
}
}
private void DoScrape(int letterCount, string source, bool doContinueWithLastWord)
{
var dbContextFactory = new DesignTimeDbContextFactory();
using (var db = dbContextFactory.CreateDbContext(connectionString, Log.Logger))
{
string lastWordString = null;
if (doContinueWithLastWord)
{
lastWordString = WordDatabaseService.GetLastWordFromLetterCount(db, source, letterCount);
}
// if we didn't get back a word, use a pattern instead
if (lastWordString == null)
{
switch (letterCount)
{
case 1:
lastWordString = "a";
break;
case 2:
lastWordString = "aa";
break;
default:
lastWordString = "aa" + new string('?', letterCount - 2);
break;
}
Log.Information("Could not find any words having '{0}' letters. Therefore using last word pattern '{1}'.", letterCount, lastWordString);
}
// Note!
// the user needs to be added before we disable tracking and disable AutoDetectChanges
// otherwise this will crash
// set admin user
var adminUser = new User()
{
FirstName = "",
LastName = "Admin",
UserName = "******"
};
// check if user already exists
var existingUser = db.DictionaryUsers.Where(u => u.FirstName == adminUser.FirstName).FirstOrDefault();
if (existingUser != null)
{
adminUser = existingUser;
}
else
{
db.DictionaryUsers.Add(adminUser);
db.SaveChanges();
}
// disable tracking to speed things up
// note that this doesn't load the virtual properties, but loads the object ids after a save
db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
// this doesn't seem to work when adding new users all the time
db.ChangeTracker.AutoDetectChangesEnabled = false;
using (var driver = ChromeDriverUtils.GetChromeDriver(true))
{
// read all words with the letter count
ReadWordsByWordPermutations(letterCount, driver, db, adminUser, lastWordString);
}
}
}
private void DoScrape(string siteUsername, string sitePassword, int startLetterCount, int endLetterCount, string source, bool doContinueWithLastWord, bool isScraperSwarm)
{
var dbContextFactory = new DesignTimeDbContextFactory();
using (var db = dbContextFactory.CreateDbContext(connectionString, Log.Logger))
{
// Note!
// the user needs to be added before we disable tracking and disable AutoDetectChanges
// otherwise this will crash
// set admin user
var adminUser = new User()
{
FirstName = "",
LastName = "Admin",
UserName = "******"
};
// check if user already exists
var existingUser = db.DictionaryUsers.Where(u => u.FirstName == adminUser.FirstName).FirstOrDefault();
if (existingUser != null)
{
adminUser = existingUser;
}
else
{
db.DictionaryUsers.Add(adminUser);
db.SaveChanges();
}
// disable tracking to speed things up
// note that this doesn't load the virtual properties, but loads the object ids after a save
db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
// this doesn't seem to work when adding new users all the time
db.ChangeTracker.AutoDetectChangesEnabled = false;
#if DEBUG
// some patterns give back a word with one less character than asked for - it seems the Ø is messing their system up
// UTF8 two byte problem?
// TROND?K????? gives TROND KJØLL
// VEBJØRN?B???? gives VEBJØRN BERG
// WILLY?R???????? gives WILLY RØGEBERG
// THORBJØRN?H??????? gives THORBJØRN HÅRSTAD
// lastWordString = "TRONSMOS VEG"; // word before TROND KJØLL
// letterCount = 12;
// lastWordString = "ÅSTED FOR DRAMAET ROMEO OG JULIE";
// letterCount = 32;
// lastWordString = "GUTTENAVN PÅ \"A\"";
// letterCount = 16;
// endLetterCount = 17;
// lastWordString = "TALL SOM ANGIR FORHOLDET MELLOM ET LEGEMES HASTIGHET OG LYDENS";
// lastWordString = "ÅPNINGSKONSERTSTYKKE";
// letterCount = lastWordString.Length;
// endLetterCount = 300;
#endif
using (var driver = ChromeDriverUtils.GetChromeDriver(true))
{
DoLogon(driver, siteUsername, sitePassword);
for (int i = startLetterCount; i < endLetterCount; i++)
{
// reset global variables
hasFoundPattern = false; // this is the first stage, we match the pattern
hasFoundLastWord = false; // this is the second stage, we not only match the pattern but the word as well
hasMissedLastWord = false;
string lastWordString = null;
if (doContinueWithLastWord)
{
lastWordString = WordDatabaseService.GetLastWordFromLetterCount(db, source, i);
}
// don't skip any words when the last word is empty
if (lastWordString == null)
{
hasFoundLastWord = true;
}
// added break to support several docker instances scraping in swarms
if (isScraperSwarm && (i > startLetterCount))
{
Log.Error("Warning! Quitting since the current letter length > letter count: {0} / {1}", i, startLetterCount);
break;
}
ReadWordsByWordPermutations(i, driver, db, adminUser, lastWordString);
}
}
}
}
