private void DoScrape(string siteUsername, string sitePassword, string source) { var dbContextFactory = new DesignTimeDbContextFactory(); using (var db = dbContextFactory.CreateDbContext(connectionString, Log.Logger)) { // Note! // the user needs to be added before we disable tracking and disable AutoDetectChanges // otherwise this will crash // set admin user var adminUser = new User() { FirstName = "", LastName = "Admin", UserName = "******" }; // check if user already exists var existingUser = db.DictionaryUsers.Where(u => u.FirstName == adminUser.FirstName).FirstOrDefault(); if (existingUser != null) { adminUser = existingUser; } else { db.DictionaryUsers.Add(adminUser); db.SaveChanges(); } // disable tracking to speed things up // note that this doesn't load the virtual properties, but loads the object ids after a save db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; // this doesn't seem to work when adding new users all the time db.ChangeTracker.AutoDetectChangesEnabled = false; using (var driver = ChromeDriverUtils.GetChromeDriver(true)) { DoLogon(driver, siteUsername, sitePassword); string url = "https://www.kryssord.org"; driver.Navigate().GoToUrl(url); var documentNode = driver.GetDocumentNode(); ProcessWordsUntilEmpty(driver, db, adminUser, documentNode); } } }
private void DoScrape(int startLetterCount, int endLetterCount, string source, bool doContinueWithLastWord) { var dbContextFactory = new DesignTimeDbContextFactory(); using (var db = dbContextFactory.CreateDbContext(connectionString, Log.Logger)) { // Note! // the user needs to be added before we disable tracking and disable AutoDetectChanges // otherwise this will crash // set admin user var adminUser = new User() { FirstName = "", LastName = "Admin", UserName = "******" }; // check if user already exists var existingUser = db.DictionaryUsers.Where(u => u.FirstName == adminUser.FirstName).FirstOrDefault(); if (existingUser != null) { adminUser = existingUser; } else { db.DictionaryUsers.Add(adminUser); db.SaveChanges(); } // disable tracking to speed things up // note that this doesn't load the virtual properties, but loads the object ids after a save db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; // this doesn't seem to work when adding new users all the time db.ChangeTracker.AutoDetectChangesEnabled = false; using (var driver = ChromeDriverUtils.GetChromeDriver(true)) { // set general timeout to long driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(180); // read all words with the letter count ReadWordsByWordPermutations(startLetterCount, endLetterCount, driver, db, adminUser, doContinueWithLastWord); } } }
private void DoScrape(int letterCount, string source, bool doContinueWithLastWord) { var dbContextFactory = new DesignTimeDbContextFactory(); using (var db = dbContextFactory.CreateDbContext(connectionString, Log.Logger)) { string lastWordString = null; if (doContinueWithLastWord) { lastWordString = WordDatabaseService.GetLastWordFromLetterCount(db, source, letterCount); } // if we didn't get back a word, use a pattern instead if (lastWordString == null) { switch (letterCount) { case 1: lastWordString = "a"; break; case 2: lastWordString = "aa"; break; default: lastWordString = "aa" + new string('?', letterCount - 2); break; } Log.Information("Could not find any words having '{0}' letters. Therefore using last word pattern '{1}'.", letterCount, lastWordString); } // Note! // the user needs to be added before we disable tracking and disable AutoDetectChanges // otherwise this will crash // set admin user var adminUser = new User() { FirstName = "", LastName = "Admin", UserName = "******" }; // check if user already exists var existingUser = db.DictionaryUsers.Where(u => u.FirstName == adminUser.FirstName).FirstOrDefault(); if (existingUser != null) { adminUser = existingUser; } else { db.DictionaryUsers.Add(adminUser); db.SaveChanges(); } // disable tracking to speed things up // note that this doesn't load the virtual properties, but loads the object ids after a save db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; // this doesn't seem to work when adding new users all the time db.ChangeTracker.AutoDetectChangesEnabled = false; using (var driver = ChromeDriverUtils.GetChromeDriver(true)) { // read all words with the letter count ReadWordsByWordPermutations(letterCount, driver, db, adminUser, lastWordString); } } }
private void DoScrape(string siteUsername, string sitePassword, int startLetterCount, int endLetterCount, string source, bool doContinueWithLastWord, bool isScraperSwarm) { var dbContextFactory = new DesignTimeDbContextFactory(); using (var db = dbContextFactory.CreateDbContext(connectionString, Log.Logger)) { // Note! // the user needs to be added before we disable tracking and disable AutoDetectChanges // otherwise this will crash // set admin user var adminUser = new User() { FirstName = "", LastName = "Admin", UserName = "******" }; // check if user already exists var existingUser = db.DictionaryUsers.Where(u => u.FirstName == adminUser.FirstName).FirstOrDefault(); if (existingUser != null) { adminUser = existingUser; } else { db.DictionaryUsers.Add(adminUser); db.SaveChanges(); } // disable tracking to speed things up // note that this doesn't load the virtual properties, but loads the object ids after a save db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; // this doesn't seem to work when adding new users all the time db.ChangeTracker.AutoDetectChangesEnabled = false; #if DEBUG // some patterns give back a word with one less character than asked for - it seems the Ø is messing their system up // UTF8 two byte problem? // TROND?K????? gives TROND KJØLL // VEBJØRN?B???? gives VEBJØRN BERG // WILLY?R???????? gives WILLY RØGEBERG // THORBJØRN?H??????? gives THORBJØRN HÅRSTAD // lastWordString = "TRONSMOS VEG"; // word before TROND KJØLL // letterCount = 12; // lastWordString = "ÅSTED FOR DRAMAET ROMEO OG JULIE"; // letterCount = 32; // lastWordString = "GUTTENAVN PÅ \"A\""; // letterCount = 16; // endLetterCount = 17; // lastWordString = "TALL SOM ANGIR FORHOLDET MELLOM ET LEGEMES HASTIGHET OG LYDENS"; // lastWordString = "ÅPNINGSKONSERTSTYKKE"; // letterCount = lastWordString.Length; // endLetterCount = 300; #endif using (var driver = ChromeDriverUtils.GetChromeDriver(true)) { DoLogon(driver, siteUsername, sitePassword); for (int i = startLetterCount; i < endLetterCount; i++) { // reset global variables hasFoundPattern = false; // this is the first stage, we match the pattern hasFoundLastWord = false; // this is the second stage, we not only match the pattern but the word as well hasMissedLastWord = false; string lastWordString = null; if (doContinueWithLastWord) { lastWordString = WordDatabaseService.GetLastWordFromLetterCount(db, source, i); } // don't skip any words when the last word is empty if (lastWordString == null) { hasFoundLastWord = true; } // added break to support several docker instances scraping in swarms if (isScraperSwarm && (i > startLetterCount)) { Log.Error("Warning! Quitting since the current letter length > letter count: {0} / {1}", i, startLetterCount); break; } ReadWordsByWordPermutations(i, driver, db, adminUser, lastWordString); } } } }