コード例 #1
0
 public WordController(IConfiguration config, UserManager <ApplicationUser> userManager, WordHintDbContext db, IApiDescriptionGroupCollectionProvider apiExplorer)
 {
     this.config      = config;
     this.userManager = userManager;
     this.db          = db;
     this.apiExplorer = apiExplorer;
 }
コード例 #2
0
 public AccountController(IConfiguration config, UserManager <ApplicationUser> userManager, WordHintDbContext db, IMapper mapper, ITokenService tokenService)
 {
     this.config       = config;
     this.userManager  = userManager;
     this.db           = db;
     this.mapper       = mapper;
     this.tokenService = tokenService;
 }
コード例 #3
0
 public CrossWordController(IConfiguration config,
                            UserManager <ApplicationUser> userManager,
                            WordHintDbContext db,
                            ICrossDictionary dictionary,
                            IBackgroundTaskQueue queue,
                            ILogger <CrossWordController> logger,
                            IServiceScopeFactory serviceScopeFactory)
 {
     this.config              = config;
     this.userManager         = userManager;
     this.db                  = db;
     this.dictionary          = dictionary;
     this.logger              = logger;
     this.Queue               = queue;
     this.serviceScopeFactory = serviceScopeFactory;
 }
コード例 #4
0
        private void ProcessWordsUntilEmpty(IWebDriver driver, WordHintDbContext db, User adminUser, HtmlNode documentNode)
        {
            Log.Information("Looking for latest words and synonyms on frontpage");
            writer.WriteLine("Looking for latest words and synonyms on frontpage");

            // parse all words
            var wordAndHrefs = ParseWordsAgilityPack(documentNode, adminUser);

            foreach (var wordAndHref in wordAndHrefs)
            {
                var wordText = wordAndHref.Item1;
                var url      = wordAndHref.Item2;

                Log.Information("Processing word found on frontpage: {0}, {1}", wordText, url);
                writer.WriteLine("Processing word found on frontpage: {0}, {1}", wordText, url);

                var word = ReadWordByWordText(driver, adminUser, wordText);
                if (word != null)
                {
                    GetWordSynonyms(word, driver, db, adminUser);
                }
            }
        }
コード例 #5
0
        private void GetWordSynonyms(Word word, IWebDriver driver, WordHintDbContext db, User adminUser, string url)
        {
            // there is a bug in the website that makes a  query with "0" fail
            if (word.Value == "0")
            {
                return;
            }

            // open a new tab and set the context
            var chromeDriver = (ChromeDriver)driver;

            // save a reference to our original tab's window handle
            var originalTabInstance = chromeDriver.CurrentWindowHandle;

            // execute some JavaScript to open a new window
            chromeDriver.ExecuteScript("window.open();");

            // save a reference to our new tab's window handle, this would be the last entry in the WindowHandles collection
            var newTabInstance = chromeDriver.WindowHandles[driver.WindowHandles.Count - 1];

            // switch our WebDriver to the new tab's window handle
            chromeDriver.SwitchTo().Window(newTabInstance);

            // lets navigate to a web site in our new tab
            driver.Navigate().GoToUrl(url);

            Log.Information("Processing synonym search for '{0}'", word.Value);
            writer.WriteLine("Processing synonym search for '{0}'", word.Value);

            // parse all synonyms
            IList <IWebElement> listElements = driver.FindElements(By.XPath("//div[@id='wordlist']/ul[@class='word']/li"));
            IWebElement         ahref        = null;

            var relatedWords = new List <Word>();

            foreach (IWebElement listElement in listElements)
            {
                try
                {
                    ahref = listElement.FindElement(By.TagName("a"));
                }
                catch (NoSuchElementException)
                {
                    break;
                }

                var hintText = ahref.Text;
                var href     = ahref.GetAttribute("href");

                var hint = new Word
                {
                    Language        = "no",
                    Value           = hintText,
                    NumberOfLetters = hintText.Count(c => c != ' '),
                    NumberOfWords   = ScraperUtils.CountNumberOfWords(hintText),
                    User            = adminUser,
                    CreatedDate     = DateTime.Now,
                    Source          = this.source
                };

                relatedWords.Add(hint);
            }

            relatedWords = relatedWords.Distinct().ToList(); // Note that this requires the object to implement IEquatable<Word>

            // and add to database
            WordDatabaseService.AddToDatabase(db, this.source, word, relatedWords, writer);

            // now lets close our new tab
            chromeDriver.ExecuteScript("window.close();");

            // and switch our WebDriver back to the original tab's window handle
            chromeDriver.SwitchTo().Window(originalTabInstance);

            // and have our WebDriver focus on the main document in the page to send commands to
            chromeDriver.SwitchTo().DefaultContent();
        }
コード例 #6
0
        private void ReadWordsByWordPattern(string wordPattern, IWebDriver driver, WordHintDbContext db, User adminUser)
        {
            // go to search page
            string url = "https://kryssordhjelp.no/";

            driver.Navigate().GoToUrl(url);

            Log.Information("Processing pattern search for '{0}'", wordPattern);
            writer.WriteLine("Processing pattern search for '{0}'", wordPattern);

            // select the drop down list
            var lengthElement = driver.FindElement(By.Name("length"));

            // create select element object
            var selectElement = new SelectElement(lengthElement);

            // select by value
            selectElement.SelectByValue(wordPattern.Length.ToString());

            // select the letter fields
            var letter1 = driver.FindElement(By.Name("letter[1]"));

            letter1.SendKeys(wordPattern[0].ToString());
            var letter2 = driver.FindElement(By.Name("letter[2]"));

            letter2.SendKeys(wordPattern[1].ToString());

            // find submit button
            var login = driver.FindElement(By.Id("submitsearch"));

            login.Click();

            // wait until the word list has loaded
            try
            {
                driver.WaitForElementLoad(By.XPath("//div[@id='wordlist']/ul[@class='word']/li"), 20);
            }
            catch (System.Exception)
            {
                Log.Error("Timeout searching for '{0}'", wordPattern);
                writer.WriteLine("Timeout searching for '{0}'", wordPattern);
                return;
            }

            // parse all words
            IList <IWebElement> listElements = driver.FindElements(By.XPath("//div[@id='wordlist']/ul[@class='word']/li"));
            IWebElement         ahref        = null;

            foreach (IWebElement listElement in listElements)
            {
                try
                {
                    ahref = listElement.FindElement(By.TagName("a"));
                }
                catch (NoSuchElementException)
                {
                    break;
                }

                var wordText = ahref.Text;
                var href     = ahref.GetAttribute("href");

                var word = new Word
                {
                    Language        = "no",
                    Value           = wordText,
                    NumberOfLetters = wordText.Count(c => c != ' '),
                    NumberOfWords   = ScraperUtils.CountNumberOfWords(wordText),
                    User            = adminUser,
                    CreatedDate     = DateTime.Now,
                    Source          = this.source
                };

                GetWordSynonyms(word, driver, db, adminUser, href);
            }
        }
コード例 #7
0
        private void ReadWordsByWordPermutations(int letterCount, IWebDriver driver, WordHintDbContext db, User adminUser, string lastWord)
        {
            int permutationSize = letterCount > 1 ? 2 : 1;

            var alphabet     = "abcdefghijklmnopqrstuvwxyzøæå";
            var permutations = alphabet.Select(x => x.ToString());

            for (int i = 0; i < permutationSize - 1; i++)
            {
                permutations = permutations.SelectMany(x => alphabet, (x, y) => x + y);
            }

            bool hasFoundWord = false;

            foreach (var permutation in permutations)
            {
                string wordPattern = "";
                if (letterCount > permutationSize)
                {
                    // make word search pattern
                    wordPattern = permutation.PadRight(letterCount, '?');
                }
                else
                {
                    wordPattern = permutation;
                }

                // if lastWord isn't null
                if (lastWord != null)
                {
                    // skip until lastWord is found
                    if (!hasFoundWord)
                    {
                        if (lastWord.Length > wordPattern.Length)
                        {
                            // skip until same length
                            continue;
                        }
                        else if (lastWord.Length < wordPattern.Length)
                        {
                            // this means we are continuing with longer patterns
                            // and should process as normal
                            hasFoundWord = true;
                        }
                        else
                        {
                            // same length so compare
                            var   patternRegexp = wordPattern.Replace('?', '.');
                            Match match         = Regex.Match(lastWord, patternRegexp, RegexOptions.IgnoreCase);
                            if (match.Success)
                            {
                                // found
                                hasFoundWord = true;
                            }
                            else
                            {
                                continue;
                            }
                        }
                    }

                    if (hasFoundWord)
                    {
                        ReadWordsByWordPattern(wordPattern, driver, db, adminUser);
                    }
                }
                else
                {
                    ReadWordsByWordPattern(wordPattern, driver, db, adminUser);
                }
            }
        }
コード例 #8
0
        private void ReadWordsByWordPermutations(int startLetterCount, int endLetterCount, IWebDriver driver, WordHintDbContext db, User adminUser, bool doContinueWithLastWord)
        {
            var alphabet        = "abcdefghijklmnopqrstuvwxyzåæøö";
            var permutations    = alphabet.Select(x => x.ToString());
            int permutationSize = 2;

            for (int i = 0; i < permutationSize - 1; i++)
            {
                permutations = permutations.SelectMany(x => alphabet, (x, y) => x + y);
            }

            var wordPermutationList = permutations.ToList();

            wordPermutationList.Add("&");
            wordPermutationList.Add("(");
            wordPermutationList.Add(")");
            wordPermutationList.Add("+");
            wordPermutationList.Add(",");
            wordPermutationList.Add("-");
            wordPermutationList.Add("0");
            wordPermutationList.Add("1");
            wordPermutationList.Add("2");
            wordPermutationList.Add("3");
            wordPermutationList.Add("4");
            wordPermutationList.Add("5");
            wordPermutationList.Add("6");
            wordPermutationList.Add("7");
            wordPermutationList.Add("8");
            wordPermutationList.Add("9");

            // use the letter count a little bit different when it comes to the alphabetic index:
            // letterCount is the index to start with divided out on the total alphabetic index
            // e.g.
            // if letter count is between 1 - 4 of a total index length of 1000:
            // 1 is 1
            // 2 is 250
            // 3 is 500
            // 4 is 750
            int length      = wordPermutationList.Count;
            int startIndex  = (int)(((double)length / (double)endLetterCount) * (startLetterCount - 1));
            int endIndex    = (int)((((double)length / (double)endLetterCount) * startLetterCount) - 1);
            var startString = wordPermutationList[startIndex];
            var endString   = wordPermutationList[endIndex];

            Log.Information("Processing alphabetic permutation search using {0}-{1} = {2}-{3} ({4} - {5}) ", startLetterCount, endLetterCount, startIndex, endIndex, startString, endString);
            writer.WriteLine("Processing alphabetic permutation search using {0}-{1} = {2}-{3} ({4} - {5}) ", startLetterCount, endLetterCount, startIndex, endIndex, startString, endString);

            // add some extra status information to the writer
            if (this.writer is SignalRClientWriter)
            {
                (this.writer as SignalRClientWriter).ExtraStatusInformation = string.Format("Processing alphabetic permutation search using {0}-{1} = {2}-{3} ({4} - {5}) ", startLetterCount, endLetterCount, startIndex, endIndex, startString, endString);
            }

            int curIndex = 0;

            foreach (var wordPermutation in wordPermutationList)
            {
                string wordPattern = wordPermutation.Length == 1 && wordPermutation[0] < 45 ? string.Format("%{0:X}", (int)wordPermutation[0]) : wordPermutation;
                curIndex++;

                if (curIndex < startIndex + 1)
                {
                    Log.Information("Skipping pattern '{0}' until we reach index {1}: '{2}'. [{3}/{4}]", wordPattern, startIndex, startString, curIndex, length);
                    writer.WriteLine("Skipping pattern '{0}' until we reach index {1}: '{2}'. [{3}/{4}]", wordPattern, startIndex, startString, curIndex, length);
                    continue;
                }
                else if (length != curIndex && curIndex == endIndex + 1) // stop at last index except very last character
                {
                    // reached the end - quit
                    Log.Information("Quitting because we have reached the last index to process: {0} at index {1}.", wordPattern, curIndex);
                    writer.WriteLine("Quitting because we have reached the last index to process: {0} at index {1}.", wordPattern, curIndex);
                    break;
                }

                string lastWordString = null;
                if (doContinueWithLastWord)
                {
                    lastWordString = WordDatabaseService.GetLastWordFromComment(db, source, wordPattern);
                }

                // var href = $"https://www.gratiskryssord.no/kryssordbok/?kart={wordPattern}#oppslag";
                var href = $"https://www.gratiskryssord.no/kryssordbok/alfabetisk/{wordPattern}/";
#if DEBUG
                // if (wordPermutation == "xå")
                // {
                //     wordPattern = "kå";
                //     href = $"https://www.gratiskryssord.no/kryssordbok/?kart={wordPattern}#oppslag";
                //     lastWordString = WordDatabaseService.GetLastWordFromComment(db, source, wordPattern);
                // }
                // else if (wordPermutation == "&")
                // {
                //     // debugging - break here
                // }

                wordPattern    = "na";
                href           = $"https://www.gratiskryssord.no/kryssordbok/alfabetisk/{wordPattern}/";
                lastWordString = "NAVN";
#endif
                ReadWordsByWordUrl(wordPattern, href, driver, db, adminUser, lastWordString);
            }
        }
コード例 #9
0
        private void GetWordSynonyms(Word word, IWebDriver driver, WordHintDbContext db, User adminUser, string url)
        {
            // there is a bug in the website that makes a  query with "0" fail
            if (word.Value == "0")
            {
                return;
            }

            // open a new tab and set the context
            var chromeDriver = (ChromeDriver)driver;

            // save a reference to our original tab's window handle
            var originalTabInstance = chromeDriver.CurrentWindowHandle;

            // execute some JavaScript to open a new window
            chromeDriver.ExecuteScript("window.open();");

            // save a reference to our new tab's window handle, this would be the last entry in the WindowHandles collection
            var newTabInstance = chromeDriver.WindowHandles[driver.WindowHandles.Count - 1];

            // switch our WebDriver to the new tab's window handle
            chromeDriver.SwitchTo().Window(newTabInstance);

            // lets navigate to a web site in our new tab
            // https://www.gratiskryssord.no/kryssordbok/navn/side/1/
            var page    = 1;
            var pageUrl = $"{url}side/{page}/";

            driver.Navigate().GoToUrl(pageUrl);
            while (true)
            {
                Log.Information("Processing synonym search for '{0}' on page {1}", word.Value, page);
                writer.WriteLine("Processing synonym search for '{0}' on page {1}", word.Value, page);

                // read the whole document into a HtmlNode
                HtmlNode doc = driver.GetDocumentNode();

                // and parse synonyms using Agility Pack
                var relatedWords = ParseSynonymsAgilityPack(word, doc, adminUser);

                // and add to database
                WordDatabaseService.AddToDatabase(db, this.source, word, relatedWords, writer, false);

                // go to next page if exist
                var nextPageElement = FindNextPageOrNull(doc, word.Value.ToLower(), page + 1);
                if (nextPageElement != null)
                {
                    var hintText = nextPageElement.InnerText.Trim().ToUpper();
                    hintText = HttpUtility.HtmlDecode(hintText); // ensure that text like &amp; gets converted to &
                    var    href        = nextPageElement.Attributes["href"].Value;
                    string nextPageUrl = $"https://www.gratiskryssord.no{href}";

                    page++;
                    driver.Navigate().GoToUrl(nextPageUrl);
                }
                else
                {
                    break;
                }
            }

            // now lets close our new tab
            chromeDriver.ExecuteScript("window.close();");

            // and switch our WebDriver back to the original tab's window handle
            chromeDriver.SwitchTo().Window(originalTabInstance);

            // and have our WebDriver focus on the main document in the page to send commands to
            chromeDriver.SwitchTo().DefaultContent();
        }
コード例 #10
0
        private void ReadWordsIntoDatabase(WordHintDbContext db)
        {
#if DEBUG
            // Create new stopwatch.
            Stopwatch stopwatch = new Stopwatch();

            // Begin timing.
            stopwatch.Start();
#endif

            // var wordIdsToExclude = WordDatabaseService.GetWordIdList(db, new List<string> { "BY", "NAVN", "ELV", "FJELL", "FORKORTELSE", "IATA-FLYPLASSKODE", "ISO-KODE" });
            var wordIdsToExclude = WordDatabaseService.GetWordIdList(db, new List <string> {
                "BY", "NAVN"
            });

            // search for all words
            var words = db.Words
                        .Where((w => (w.NumberOfWords == 1) && (w.NumberOfLetters <= _maxWordLength) && !wordIdsToExclude.Contains(w.WordId)))
                        .OrderBy(w => w.Value)
                        .Select(w => w.Value)
                        .AsNoTracking();

            // search for all words
            // var words = _db.Words
            //     .Where((w => (w.NumberOfWords == 1) && (w.NumberOfLetters <= _maxWordLength)))
            //     .OrderBy(w => w.Value)
            //     .Select(w => w.Value)
            //     .AsNoTracking();

            // in order to sort with Collation we need to use raw SQL
            // var words = _db.Words.FromSql(
            //     $"SELECT w.Value FROM Words AS w WHERE w.NumberOfWords = 1 AND w.NumberOfLetters <= {_maxWordLength} ORDER BY w.Value COLLATE utf8mb4_da_0900_as_cs")
            //     .Select(w => w.Value)
            //     .AsNoTracking();

            foreach (var word in words)
            {
                string wordText = word;
                if (wordText.All(char.IsLetter))
                // if (wordText.All(x => char.IsLetter(x) || x == '-' || x == ' '))
                {
                    AddWord(wordText);
                }
            }

            // using ADO.NET seems faster than ef core for raw SQLs
            // using (var command = _db.Database.GetDbConnection().CreateCommand())
            // {
            //     command.CommandText = $"SELECT w.Value FROM Words AS w WHERE w.NumberOfWords = 1 AND w.NumberOfLetters <= {_maxWordLength} ORDER BY w.Value COLLATE utf8mb4_da_0900_as_cs";
            //     db.Database.OpenConnection();
            //     using (var reader = command.ExecuteReader())
            //     {
            //         while (reader.Read())
            //         {
            //             string wordText = reader[0].ToString();
            //             if (wordText.All(char.IsLetter))
            //             // if (wordText.All(x => char.IsLetter(x) || x == '-' || x == ' '))
            //             {
            //                 AddWord(wordText);
            //             }
            //         }
            //     }
            // }

#if DEBUG
            // Stop timing.
            stopwatch.Stop();

            // Write result.
            if (_logger != null)
            {
                _logger.LogDebug("ReadWordsIntoDatabase - Time elapsed: {0}", stopwatch.Elapsed);
            }
            else
            {
                Console.WriteLine("ReadWordsIntoDatabase - Time elapsed: {0}", stopwatch.Elapsed);
            }
#endif
        }
コード例 #11
0
ファイル: KryssordScraper.cs プロジェクト: perivar/CrossWord
        private void ProcessWordsUntilEmpty(WordPattern wordPattern, IWebDriver driver, WordHintDbContext db, User adminUser, int page, HtmlNode documentNode, string url)
        {
            while (true)
            {
                Log.Information("Processing pattern search for '{0}' on page {1}", wordPattern.Pattern, page + 1);
                writer.WriteLine("Processing pattern search for '{0}' on page {1}", wordPattern.Pattern, page + 1);

                // parse all words
                var words = ReadWordsAgilityPack(documentNode, adminUser);

                foreach (var word in words)
                {
                    if (wordPattern.IsMatchLastWord)
                    {
                        Log.Information("The current pattern matches the last-word: {0} = {1}. Current word: {2}", wordPattern.Pattern, wordPattern.LastWord, word.Value);
                        hasFoundPattern = true;

                        var wordRemoveDiacriticsToNorwegian = word.Value.RemoveDiacriticsToNorwegian();

                        // we might have had to add question marks at the end of the string to fix the length bug at the site
                        if (wordRemoveDiacriticsToNorwegian == wordPattern.LastWord.TrimEnd('?'))
                        {
                            Log.Information("The current word matches the last-word: {0} = {1}", word.Value, wordPattern.LastWord);
                            hasFoundLastWord = true;
                        }
                    }
                    else
                    {
                        if (!hasFoundLastWord && hasFoundPattern)
                        {
                            // if the pattern not any longer match, we never found the word - has it been deleted?
                            Log.Error("Warning! The current pattern does not any longer match the last-word: {0} = {1}. Current word: {2}", wordPattern.Pattern, wordPattern.LastWord, word.Value);
                            writer.WriteLine("Warning! The current pattern does not any longer match the last-word: {0} = {1}. Current word: {2}", wordPattern.Pattern, wordPattern.LastWord, word.Value);
                            hasMissedLastWord = true;
                            return;
                        }
                    }

                    if (hasFoundLastWord)
                    {
                        string currentValue = word.Value;

                        // check if this is one of the buggy words from their site where the words found don't have the same length as the pattern says it should have
                        if (wordPattern.Length != word.Value.Length)
                        {
                            Log.Error("Warning! The current word doesn't match the length of the query pattern: {0} = {1}", word.Value, wordPattern.Pattern);
                            writer.WriteLine("Warning! The current word doesn't match the length of the query pattern: {0} = {1}", word.Value, wordPattern.Pattern);

                            if (wordPattern.Length > word.Value.Length)
                            {
                                currentValue = currentValue + new string('?', wordPattern.Length - word.Value.Length);
                            }
                            else
                            {
                                currentValue = currentValue.Substring(0, wordPattern.Length);
                            }
                        }
                        else
                        {
                            // everything is OK
                        }

                        // update that we are processing this word, ignore length and comment
                        WordDatabaseService.UpdateState(db, source, new Word()
                        {
                            Value = currentValue.ToUpper(), Source = source, CreatedDate = DateTime.Now
                        }, writer);

                        GetWordSynonyms(word, driver, db, adminUser);
                    }
                }

                // go to next page if exist
                var(hasFoundNextPage, pageNumber, pageUrl, pageNode) = NavigateToNextPageIfExist(driver, documentNode);
                if (hasFoundNextPage)
                {
                    url          = pageUrl;
                    page         = pageNumber;
                    documentNode = pageNode;
                }
                else
                {
                    break;
                }
            }
        }
コード例 #12
0
ファイル: KryssordScraper.cs プロジェクト: perivar/CrossWord
        // recursive function that progressively increases the depth if we get a too high word count
        // but maximum 3 permutations (abc???)
        private void ReadWordsByWordPermutationsRecursive(IWebDriver driver, WordPattern wordPattern, WordHintDbContext db, User adminUser)
        {
            // get word count
            var(wordCount, node, url, page) = GetWordCountByPattern(driver, wordPattern);

            if (wordCount == 0)
            {
                return;
            }
            else
            {
                Log.Information("Found {0} words when searching for '{1}' on page {2}", wordCount, wordPattern.Pattern, page + 1);
                writer.WriteLine("Found {0} words when searching for '{1}' on page {2}", wordCount, wordPattern.Pattern, page + 1);
            }

            // if we get too many words back, try to increase the pattern depth
            // but maximum 4 levels
            if (wordCount <= 108 || wordPattern.Depth > 3)
            {
                if (wordPattern.Depth > 3)
                {
                    Log.Error("Warning! Pattern search depth is now {0}. Found {1} words when searching for '{2}' on page {3}", wordPattern.Depth, wordCount, wordPattern.Pattern, page + 1);
                }

                // process each word found using the specified word pattern
                ProcessWordsUntilEmpty(wordPattern, driver, db, adminUser, page, node, url);
            }
            else
            {
                // increment pattern
                var childPatterns = wordPattern.GetWordPatternChildren();

                // recursively process children patterns
                foreach (var childPattern in childPatterns)
                {
                    if (hasMissedLastWord)
                    {
                        return;
                    }

                    // if we have a last word - make sure to skip until the pattern matches
                    if (!hasFoundLastWord && childPattern.LastWord != null && !childPattern.IsMatchLastWord)
                    {
                        // skip pattern
                        Log.Information("Skipping pattern '{0}'.", childPattern.Pattern);
                    }
                    else
                    {
                        Log.Information("Processing pattern '{0}'.", childPattern.Pattern);
                        ReadWordsByWordPermutationsRecursive(driver, childPattern, db, adminUser);
                    }
                }
            }
        }
コード例 #13
0
ファイル: KryssordScraper.cs プロジェクト: perivar/CrossWord
        private void ReadWordsByWordPermutations(int letterLength, IWebDriver driver, WordHintDbContext db, User adminUser, string lastWord)
        {
            int wordLength = letterLength;

            int depth = 1;

            // if we are using only one letter, set the depth to zero to force the pattern to become ?
            if (letterLength == 1)
            {
                depth = 0;
            }

            // set wordlength based on last word if we are not processing in swarm mode and are continuing
            // to read patterns with increasing length
            // this assumes that we found the last word for the first pattern match
            if (!hasFoundLastWord && lastWord != null)
            {
                wordLength = lastWord.Length;

                // disabling this as it doesn't give any real benefits - only misses words that start with one character for high letter counts
                // if the word is longer than 3 characters, use the two first letters as pattern
                // if (wordLength > 2)
                // {
                //     depth = 2;
                // }
            }

            var permutations = GetPermutations(depth);


            bool hasFound = false;

            foreach (var permutation in permutations)
            {
                if (hasMissedLastWord)
                {
                    return;
                }

                // skip until we reach last word beginning
                if (!hasFoundLastWord && lastWord != null)
                {
                    if (lastWord.ToLowerInvariant().StartsWith(permutation))
                    {
                        hasFound = true;
                    }
                }
                else
                {
                    hasFound = true;
                }

                if (hasFound)
                {
                    ReadWordsByWordPermutationsRecursive(driver, new WordPattern(permutation, wordLength, depth, lastWord), db, adminUser);
                }
            }
        }
コード例 #14
0
        private void ReadDescriptionsIntoDatabase(WordHintDbContext db, List <string> words)
        {
#if DEBUG
            // Create new stopwatch.
            Stopwatch stopwatch = new Stopwatch();

            // Begin timing.
            stopwatch.Start();
#endif

            Random rnd = new Random();

            // find out which words have already a description
            var newWords = words.Where(value => !_description.Any(entry => entry.Key == value));

            // find the words in the database
            var existingWords = db.Words
                                .Include(w => w.RelatedFrom)
                                .ThenInclude(w => w.WordTo)
                                .Include(w => w.RelatedTo)
                                .ThenInclude(w => w.WordFrom)
                                .Where(x => newWords.Contains(x.Value))
                                .AsNoTracking();

            foreach (var word in existingWords)
            {
                var wordText = word.Value;

                if (word.RelatedFrom.Count > 0)
                {
                    // string hintText = word.RelatedFrom.Last().WordTo.Value

                    // randomize the word descriptions
                    int    indexFrom = rnd.Next(0, word.RelatedFrom.Count - 1);
                    string hintText  = word.RelatedFrom.ToArray()[indexFrom].WordTo.Value;

                    AddDescription(wordText, hintText);
                }
                else if (word.RelatedTo.Count > 0)
                {
                    // string hintText = word.RelatedTo.Last().WordFrom.Value;

                    // randomize the word descriptions
                    int    indexTo  = rnd.Next(0, word.RelatedTo.Count - 1);
                    string hintText = word.RelatedTo.ToArray()[indexTo].WordFrom.Value;

                    AddDescription(wordText, hintText);
                }
            }

#if DEBUG
            // Stop timing.
            stopwatch.Stop();

            // Write result.
            if (_logger != null)
            {
                _logger.LogDebug("ReadDescriptionsIntoDatabase - Time elapsed: {0}", stopwatch.Elapsed);
            }
            else
            {
                Console.WriteLine("ReadDescriptionsIntoDatabase - Time elapsed: {0}", stopwatch.Elapsed);
            }
#endif
        }
コード例 #15
0
ファイル: Startup.cs プロジェクト: perivar/CrossWord
        // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
        public void Configure(IApplicationBuilder app, IHostingEnvironment env, WordHintDbContext db
                              , VersionedODataModelBuilder modelBuilder
                              , IApiVersionDescriptionProvider provider
                              )
        {
            app.UseForwardedHeaders();

            if (env.IsDevelopment())
            {
                app.UseRequestResponseLogging();
                app.UseDeveloperExceptionPage();
                app.UseDatabaseErrorPage();
            }
            else
            {
                app.UseExceptionHandler("/Error");
                // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
                app.UseHsts();
            }

            // app.UseHttpsRedirection(); // disable to use within docker behind a proxy
            app.UseStaticFiles();
            app.UseCookiePolicy();

            // You would either call EnsureCreated() or Migrate().
            // EnsureCreated() is an alternative that completely skips the migrations pipeline and just creates a database that matches you current model.
            // It's good for unit testing or very early prototyping, when you are happy just to delete and re-create the database when the model changes.
            // db.Database.EnsureDeleted();
            // db.Database.EnsureCreated();

            // Note! Therefore don't use EnsureDeleted() and EnsureCreated() but Migrate();
            db.Database.Migrate();

            app.UseCors("Everything");

            app.UseAuthentication();

            // add signalr hub url
            app.UseSignalR(routes =>
            {
                routes.MapHub <CrossWordSignalRHub>("/crosswordsignalrhub");
            });

            // Add support for OData to MVC pipeline
            var models = modelBuilder.GetEdmModels(); // versioning API

            // var model = WordModelConfiguration.GetEdmModel(new ODataConventionModelBuilder()); // single odata API
            app.UseMvc(routes =>
            {
                // routes.MapRoute(
                //     name: "default",
                //     template: "{controller=Home}/{action=Index}/{id?}");

                // setup odata filters
                routes.Select().Expand().Filter().OrderBy().MaxTop(300).Count();

                // version with query parameter
                routes.MapVersionedODataRoutes(
                    routeName: "odata",
                    routePrefix: "odata",
                    models: models);

                // version by path
                // routes.MapVersionedODataRoutes(
                //     routeName: "odata-bypath",
                //     routePrefix: "odata/v{version:apiVersion}",
                //     models: models);

                // setup non-versioned odata route
                // routes.MapODataServiceRoute("odata", "odata", model);

                // Error: Cannot find the services container for the non-OData route.
                // This can occur when using OData components on the non-OData route and is usually a configuration issue.
                // Call EnableDependencyInjection() to enable OData components on non-OData routes.
                // This may also occur when a request was mistakenly handled by the ASP.NET Core routing layer instead of the OData routing layer,
                // for instance the URL does not include the OData route prefix configured via a call to MapODataServiceRoute().
                // Workaround: https://github.com/OData/WebApi/issues/1175
                // routes.EnableDependencyInjection();
            });

            // Enable middleware to serve swagger-ui (HTML, JS, CSS, etc.),
            app.UseODataVersioningSwaggerDocumentation(modelBuilder, provider);
            // app.UseSwaggerDocumentation();
        }
コード例 #16
0
        private void ReadWordsByWordUrl(string wordPrefix, string url, IWebDriver driver, WordHintDbContext db, User adminUser, string lastWord)
        {
            // go to word page
            try
            {
                driver.Navigate().GoToUrl(url);
            }
            catch (System.Exception)
            {
                // Log.Error("Timeout navigating to '{0}'", url);
                writer.WriteLine("Timeout navigating to '{0}'", url);
                return;
            }

            Log.Information("Processing word search for '{0}'", wordPrefix);
            writer.WriteLine("Processing word search for '{0}'", wordPrefix);

            // read the whole document into a HtmlNode
            HtmlNode doc = driver.GetDocumentNode();

            // and parse using agility pack
            var words = ParseWordsAgilityPack(doc, adminUser);

            bool doSkip = true;

            foreach (var wordAndHref in words)
            {
                var word     = wordAndHref.Item1;
                var href     = wordAndHref.Item2;
                var wordText = word.Value;

                // skip until we get to the last word
                if (doSkip && lastWord != null && lastWord != wordText)
                {
                    Log.Information("Skipping alphabetic word '{0}' until we find '{1}'", wordText, lastWord);
                    writer.WriteLine("Skipping alphabetic word '{0}' until we find '{1}'", wordText, lastWord);
                    continue;
                }
                doSkip = false; // make sure we don't skip on the next word after we have skipped

                // update that we are processing this word
                WordDatabaseService.UpdateState(db, source, new Word()
                {
                    Value = wordText, Comment = wordPrefix, CreatedDate = DateTime.Now
                }, writer, true);

                GetWordSynonyms(word, driver, db, adminUser, href);
            }
        }
コード例 #17
0
        private void ReadWordsFromUrl(WordHintDbContext db, User adminUser, string lastWord)
        {
            using (WebClient client = new WebClient())
                using (Stream stream = client.OpenRead(JSON_URL))
                    using (StreamReader streamReader = new StreamReader(stream))

                        using (JsonTextReader reader = new JsonTextReader(streamReader))
                        {
                            reader.SupportMultipleContent = true;

                            string        currentValue = null;
                            List <string> currentList  = null;
                            int           totalCount   = 25000;
                            int           count        = 0;

                            bool hasFound = false;

                            var serializer = new JsonSerializer();
                            while (reader.Read())
                            {
                                // output the stream one chunk at a time
                                // Log.Information(string.Format("{0,-12}  {1}",
                                //         reader.TokenType.ToString(),
                                //         reader.Value != null ? reader.Value.ToString() : "(null)"));

                                switch (reader.TokenType)
                                {
                                // JsonToken.StartObject = deserialize only when there's "{" character in the stream
                                case JsonToken.StartObject:
                                    break;

                                // JsonToken.PropertyName = deserialize only when there's a "text": in the stream
                                case JsonToken.PropertyName:
                                    currentValue = reader.Value.ToString();
                                    break;

                                // JsonToken.String = deserialize only when there's a "text" in the stream
                                case JsonToken.String:
                                    currentList.Add(reader.Value.ToString());
                                    break;

                                // JsonToken.StartArray = deserialize only when there's "[" character in the stream
                                case JsonToken.StartArray:
                                    currentList = new List <string>();
                                    break;

                                // JsonToken.EndArray = deserialize only when there's "]" character in the stream
                                case JsonToken.EndArray:
                                    count++;

                                    // skip until we reach last word beginning
                                    if (lastWord != null)
                                    {
                                        if (currentValue.ToUpperInvariant().Equals(lastWord))
                                        {
                                            hasFound = true;
                                        }
                                    }
                                    else
                                    {
                                        hasFound = true;
                                    }

                                    // store to database
                                    if (hasFound)
                                    {
                                        // update that we are processing this word, ignore length and comment
                                        WordDatabaseService.UpdateState(db, source, new Word()
                                        {
                                            Value = currentValue.ToUpper(), Source = source, CreatedDate = DateTime.Now
                                        }, writer, true);

                                        // disable storing state since we are doing it manually above
                                        WordDatabaseService.AddToDatabase(db, source, adminUser, currentValue, currentList, writer, false);

                                        // if (writer != null) writer.WriteLine("Added '{0} => {1}'", currentValue, string.Join(",", currentList));
                                        if ((count % 10) == 0)
                                        {
                                            if (writer != null)
                                            {
                                                writer.WriteLine("[{0}] / [{1}]", count, totalCount);
                                            }
                                        }
                                    }

                                    //  and reset
                                    currentList  = null;
                                    currentValue = null;
                                    break;

                                // JsonToken.EndObject = deserialize only when there's "}" character in the stream
                                case JsonToken.EndObject:
                                    currentList  = null;
                                    currentValue = null;
                                    break;
                                }
                            }
                        }

            /*
             * // reading the whole thing took approx the same time as the streaming version
             * {
             *  var json = streamReader.ReadToEnd();
             *  var jobj = JObject.Parse(json);
             *
             *  var totalCount = jobj.Properties().Count();
             *  int count = 0;
             *  foreach (var item in jobj.Properties())
             *  {
             *      count++;
             *
             *      var currentValue = item.Name;
             *      var currentList = item.Values().Select(a => a.Value<string>());
             *
             *      WordDatabaseService.AddToDatabase(db, source, adminUser, currentValue, currentList);
             *
             *      // if (writer != null) writer.WriteLine("Added '{0} => {1}'", currentValue, string.Join(",", currentList));
             *      if (writer != null) writer.WriteLine("[{0}] / [{1}]", count, totalCount);
             *  }
             * }
             */
        }