public WordController(IConfiguration config, UserManager <ApplicationUser> userManager, WordHintDbContext db, IApiDescriptionGroupCollectionProvider apiExplorer) { this.config = config; this.userManager = userManager; this.db = db; this.apiExplorer = apiExplorer; }
public AccountController(IConfiguration config, UserManager <ApplicationUser> userManager, WordHintDbContext db, IMapper mapper, ITokenService tokenService) { this.config = config; this.userManager = userManager; this.db = db; this.mapper = mapper; this.tokenService = tokenService; }
public CrossWordController(IConfiguration config, UserManager <ApplicationUser> userManager, WordHintDbContext db, ICrossDictionary dictionary, IBackgroundTaskQueue queue, ILogger <CrossWordController> logger, IServiceScopeFactory serviceScopeFactory) { this.config = config; this.userManager = userManager; this.db = db; this.dictionary = dictionary; this.logger = logger; this.Queue = queue; this.serviceScopeFactory = serviceScopeFactory; }
private void ProcessWordsUntilEmpty(IWebDriver driver, WordHintDbContext db, User adminUser, HtmlNode documentNode) { Log.Information("Looking for latest words and synonyms on frontpage"); writer.WriteLine("Looking for latest words and synonyms on frontpage"); // parse all words var wordAndHrefs = ParseWordsAgilityPack(documentNode, adminUser); foreach (var wordAndHref in wordAndHrefs) { var wordText = wordAndHref.Item1; var url = wordAndHref.Item2; Log.Information("Processing word found on frontpage: {0}, {1}", wordText, url); writer.WriteLine("Processing word found on frontpage: {0}, {1}", wordText, url); var word = ReadWordByWordText(driver, adminUser, wordText); if (word != null) { GetWordSynonyms(word, driver, db, adminUser); } } }
private void GetWordSynonyms(Word word, IWebDriver driver, WordHintDbContext db, User adminUser, string url) { // there is a bug in the website that makes a query with "0" fail if (word.Value == "0") { return; } // open a new tab and set the context var chromeDriver = (ChromeDriver)driver; // save a reference to our original tab's window handle var originalTabInstance = chromeDriver.CurrentWindowHandle; // execute some JavaScript to open a new window chromeDriver.ExecuteScript("window.open();"); // save a reference to our new tab's window handle, this would be the last entry in the WindowHandles collection var newTabInstance = chromeDriver.WindowHandles[driver.WindowHandles.Count - 1]; // switch our WebDriver to the new tab's window handle chromeDriver.SwitchTo().Window(newTabInstance); // lets navigate to a web site in our new tab driver.Navigate().GoToUrl(url); Log.Information("Processing synonym search for '{0}'", word.Value); writer.WriteLine("Processing synonym search for '{0}'", word.Value); // parse all synonyms IList <IWebElement> listElements = driver.FindElements(By.XPath("//div[@id='wordlist']/ul[@class='word']/li")); IWebElement ahref = null; var relatedWords = new List <Word>(); foreach (IWebElement listElement in listElements) { try { ahref = listElement.FindElement(By.TagName("a")); } catch (NoSuchElementException) { break; } var hintText = ahref.Text; var href = ahref.GetAttribute("href"); var hint = new Word { Language = "no", Value = hintText, NumberOfLetters = hintText.Count(c => c != ' '), NumberOfWords = ScraperUtils.CountNumberOfWords(hintText), User = adminUser, CreatedDate = DateTime.Now, Source = this.source }; relatedWords.Add(hint); } relatedWords = relatedWords.Distinct().ToList(); // Note that this requires the object to implement IEquatable<Word> // and add to database WordDatabaseService.AddToDatabase(db, this.source, word, relatedWords, writer); // now lets close our new tab chromeDriver.ExecuteScript("window.close();"); // and switch our WebDriver back to the original tab's window handle chromeDriver.SwitchTo().Window(originalTabInstance); // and have our WebDriver focus on the main document in the page to send commands to chromeDriver.SwitchTo().DefaultContent(); }
private void ReadWordsByWordPattern(string wordPattern, IWebDriver driver, WordHintDbContext db, User adminUser) { // go to search page string url = "https://kryssordhjelp.no/"; driver.Navigate().GoToUrl(url); Log.Information("Processing pattern search for '{0}'", wordPattern); writer.WriteLine("Processing pattern search for '{0}'", wordPattern); // select the drop down list var lengthElement = driver.FindElement(By.Name("length")); // create select element object var selectElement = new SelectElement(lengthElement); // select by value selectElement.SelectByValue(wordPattern.Length.ToString()); // select the letter fields var letter1 = driver.FindElement(By.Name("letter[1]")); letter1.SendKeys(wordPattern[0].ToString()); var letter2 = driver.FindElement(By.Name("letter[2]")); letter2.SendKeys(wordPattern[1].ToString()); // find submit button var login = driver.FindElement(By.Id("submitsearch")); login.Click(); // wait until the word list has loaded try { driver.WaitForElementLoad(By.XPath("//div[@id='wordlist']/ul[@class='word']/li"), 20); } catch (System.Exception) { Log.Error("Timeout searching for '{0}'", wordPattern); writer.WriteLine("Timeout searching for '{0}'", wordPattern); return; } // parse all words IList <IWebElement> listElements = driver.FindElements(By.XPath("//div[@id='wordlist']/ul[@class='word']/li")); IWebElement ahref = null; foreach (IWebElement listElement in listElements) { try { ahref = listElement.FindElement(By.TagName("a")); } catch (NoSuchElementException) { break; } var wordText = ahref.Text; var href = ahref.GetAttribute("href"); var word = new Word { Language = "no", Value = wordText, NumberOfLetters = wordText.Count(c => c != ' '), NumberOfWords = ScraperUtils.CountNumberOfWords(wordText), User = adminUser, CreatedDate = DateTime.Now, Source = this.source }; GetWordSynonyms(word, driver, db, adminUser, href); } }
private void ReadWordsByWordPermutations(int letterCount, IWebDriver driver, WordHintDbContext db, User adminUser, string lastWord) { int permutationSize = letterCount > 1 ? 2 : 1; var alphabet = "abcdefghijklmnopqrstuvwxyzøæå"; var permutations = alphabet.Select(x => x.ToString()); for (int i = 0; i < permutationSize - 1; i++) { permutations = permutations.SelectMany(x => alphabet, (x, y) => x + y); } bool hasFoundWord = false; foreach (var permutation in permutations) { string wordPattern = ""; if (letterCount > permutationSize) { // make word search pattern wordPattern = permutation.PadRight(letterCount, '?'); } else { wordPattern = permutation; } // if lastWord isn't null if (lastWord != null) { // skip until lastWord is found if (!hasFoundWord) { if (lastWord.Length > wordPattern.Length) { // skip until same length continue; } else if (lastWord.Length < wordPattern.Length) { // this means we are continuing with longer patterns // and should process as normal hasFoundWord = true; } else { // same length so compare var patternRegexp = wordPattern.Replace('?', '.'); Match match = Regex.Match(lastWord, patternRegexp, RegexOptions.IgnoreCase); if (match.Success) { // found hasFoundWord = true; } else { continue; } } } if (hasFoundWord) { ReadWordsByWordPattern(wordPattern, driver, db, adminUser); } } else { ReadWordsByWordPattern(wordPattern, driver, db, adminUser); } } }
private void ReadWordsByWordPermutations(int startLetterCount, int endLetterCount, IWebDriver driver, WordHintDbContext db, User adminUser, bool doContinueWithLastWord) { var alphabet = "abcdefghijklmnopqrstuvwxyzåæøö"; var permutations = alphabet.Select(x => x.ToString()); int permutationSize = 2; for (int i = 0; i < permutationSize - 1; i++) { permutations = permutations.SelectMany(x => alphabet, (x, y) => x + y); } var wordPermutationList = permutations.ToList(); wordPermutationList.Add("&"); wordPermutationList.Add("("); wordPermutationList.Add(")"); wordPermutationList.Add("+"); wordPermutationList.Add(","); wordPermutationList.Add("-"); wordPermutationList.Add("0"); wordPermutationList.Add("1"); wordPermutationList.Add("2"); wordPermutationList.Add("3"); wordPermutationList.Add("4"); wordPermutationList.Add("5"); wordPermutationList.Add("6"); wordPermutationList.Add("7"); wordPermutationList.Add("8"); wordPermutationList.Add("9"); // use the letter count a little bit different when it comes to the alphabetic index: // letterCount is the index to start with divided out on the total alphabetic index // e.g. // if letter count is between 1 - 4 of a total index length of 1000: // 1 is 1 // 2 is 250 // 3 is 500 // 4 is 750 int length = wordPermutationList.Count; int startIndex = (int)(((double)length / (double)endLetterCount) * (startLetterCount - 1)); int endIndex = (int)((((double)length / (double)endLetterCount) * startLetterCount) - 1); var startString = wordPermutationList[startIndex]; var endString = wordPermutationList[endIndex]; Log.Information("Processing alphabetic permutation search using {0}-{1} = {2}-{3} ({4} - {5}) ", startLetterCount, endLetterCount, startIndex, endIndex, startString, endString); writer.WriteLine("Processing alphabetic permutation search using {0}-{1} = {2}-{3} ({4} - {5}) ", startLetterCount, endLetterCount, startIndex, endIndex, startString, endString); // add some extra status information to the writer if (this.writer is SignalRClientWriter) { (this.writer as SignalRClientWriter).ExtraStatusInformation = string.Format("Processing alphabetic permutation search using {0}-{1} = {2}-{3} ({4} - {5}) ", startLetterCount, endLetterCount, startIndex, endIndex, startString, endString); } int curIndex = 0; foreach (var wordPermutation in wordPermutationList) { string wordPattern = wordPermutation.Length == 1 && wordPermutation[0] < 45 ? string.Format("%{0:X}", (int)wordPermutation[0]) : wordPermutation; curIndex++; if (curIndex < startIndex + 1) { Log.Information("Skipping pattern '{0}' until we reach index {1}: '{2}'. [{3}/{4}]", wordPattern, startIndex, startString, curIndex, length); writer.WriteLine("Skipping pattern '{0}' until we reach index {1}: '{2}'. [{3}/{4}]", wordPattern, startIndex, startString, curIndex, length); continue; } else if (length != curIndex && curIndex == endIndex + 1) // stop at last index except very last character { // reached the end - quit Log.Information("Quitting because we have reached the last index to process: {0} at index {1}.", wordPattern, curIndex); writer.WriteLine("Quitting because we have reached the last index to process: {0} at index {1}.", wordPattern, curIndex); break; } string lastWordString = null; if (doContinueWithLastWord) { lastWordString = WordDatabaseService.GetLastWordFromComment(db, source, wordPattern); } // var href = $"https://www.gratiskryssord.no/kryssordbok/?kart={wordPattern}#oppslag"; var href = $"https://www.gratiskryssord.no/kryssordbok/alfabetisk/{wordPattern}/"; #if DEBUG // if (wordPermutation == "xå") // { // wordPattern = "kå"; // href = $"https://www.gratiskryssord.no/kryssordbok/?kart={wordPattern}#oppslag"; // lastWordString = WordDatabaseService.GetLastWordFromComment(db, source, wordPattern); // } // else if (wordPermutation == "&") // { // // debugging - break here // } wordPattern = "na"; href = $"https://www.gratiskryssord.no/kryssordbok/alfabetisk/{wordPattern}/"; lastWordString = "NAVN"; #endif ReadWordsByWordUrl(wordPattern, href, driver, db, adminUser, lastWordString); } }
private void GetWordSynonyms(Word word, IWebDriver driver, WordHintDbContext db, User adminUser, string url) { // there is a bug in the website that makes a query with "0" fail if (word.Value == "0") { return; } // open a new tab and set the context var chromeDriver = (ChromeDriver)driver; // save a reference to our original tab's window handle var originalTabInstance = chromeDriver.CurrentWindowHandle; // execute some JavaScript to open a new window chromeDriver.ExecuteScript("window.open();"); // save a reference to our new tab's window handle, this would be the last entry in the WindowHandles collection var newTabInstance = chromeDriver.WindowHandles[driver.WindowHandles.Count - 1]; // switch our WebDriver to the new tab's window handle chromeDriver.SwitchTo().Window(newTabInstance); // lets navigate to a web site in our new tab // https://www.gratiskryssord.no/kryssordbok/navn/side/1/ var page = 1; var pageUrl = $"{url}side/{page}/"; driver.Navigate().GoToUrl(pageUrl); while (true) { Log.Information("Processing synonym search for '{0}' on page {1}", word.Value, page); writer.WriteLine("Processing synonym search for '{0}' on page {1}", word.Value, page); // read the whole document into a HtmlNode HtmlNode doc = driver.GetDocumentNode(); // and parse synonyms using Agility Pack var relatedWords = ParseSynonymsAgilityPack(word, doc, adminUser); // and add to database WordDatabaseService.AddToDatabase(db, this.source, word, relatedWords, writer, false); // go to next page if exist var nextPageElement = FindNextPageOrNull(doc, word.Value.ToLower(), page + 1); if (nextPageElement != null) { var hintText = nextPageElement.InnerText.Trim().ToUpper(); hintText = HttpUtility.HtmlDecode(hintText); // ensure that text like & gets converted to & var href = nextPageElement.Attributes["href"].Value; string nextPageUrl = $"https://www.gratiskryssord.no{href}"; page++; driver.Navigate().GoToUrl(nextPageUrl); } else { break; } } // now lets close our new tab chromeDriver.ExecuteScript("window.close();"); // and switch our WebDriver back to the original tab's window handle chromeDriver.SwitchTo().Window(originalTabInstance); // and have our WebDriver focus on the main document in the page to send commands to chromeDriver.SwitchTo().DefaultContent(); }
private void ReadWordsIntoDatabase(WordHintDbContext db) { #if DEBUG // Create new stopwatch. Stopwatch stopwatch = new Stopwatch(); // Begin timing. stopwatch.Start(); #endif // var wordIdsToExclude = WordDatabaseService.GetWordIdList(db, new List<string> { "BY", "NAVN", "ELV", "FJELL", "FORKORTELSE", "IATA-FLYPLASSKODE", "ISO-KODE" }); var wordIdsToExclude = WordDatabaseService.GetWordIdList(db, new List <string> { "BY", "NAVN" }); // search for all words var words = db.Words .Where((w => (w.NumberOfWords == 1) && (w.NumberOfLetters <= _maxWordLength) && !wordIdsToExclude.Contains(w.WordId))) .OrderBy(w => w.Value) .Select(w => w.Value) .AsNoTracking(); // search for all words // var words = _db.Words // .Where((w => (w.NumberOfWords == 1) && (w.NumberOfLetters <= _maxWordLength))) // .OrderBy(w => w.Value) // .Select(w => w.Value) // .AsNoTracking(); // in order to sort with Collation we need to use raw SQL // var words = _db.Words.FromSql( // $"SELECT w.Value FROM Words AS w WHERE w.NumberOfWords = 1 AND w.NumberOfLetters <= {_maxWordLength} ORDER BY w.Value COLLATE utf8mb4_da_0900_as_cs") // .Select(w => w.Value) // .AsNoTracking(); foreach (var word in words) { string wordText = word; if (wordText.All(char.IsLetter)) // if (wordText.All(x => char.IsLetter(x) || x == '-' || x == ' ')) { AddWord(wordText); } } // using ADO.NET seems faster than ef core for raw SQLs // using (var command = _db.Database.GetDbConnection().CreateCommand()) // { // command.CommandText = $"SELECT w.Value FROM Words AS w WHERE w.NumberOfWords = 1 AND w.NumberOfLetters <= {_maxWordLength} ORDER BY w.Value COLLATE utf8mb4_da_0900_as_cs"; // db.Database.OpenConnection(); // using (var reader = command.ExecuteReader()) // { // while (reader.Read()) // { // string wordText = reader[0].ToString(); // if (wordText.All(char.IsLetter)) // // if (wordText.All(x => char.IsLetter(x) || x == '-' || x == ' ')) // { // AddWord(wordText); // } // } // } // } #if DEBUG // Stop timing. stopwatch.Stop(); // Write result. if (_logger != null) { _logger.LogDebug("ReadWordsIntoDatabase - Time elapsed: {0}", stopwatch.Elapsed); } else { Console.WriteLine("ReadWordsIntoDatabase - Time elapsed: {0}", stopwatch.Elapsed); } #endif }
private void ProcessWordsUntilEmpty(WordPattern wordPattern, IWebDriver driver, WordHintDbContext db, User adminUser, int page, HtmlNode documentNode, string url) { while (true) { Log.Information("Processing pattern search for '{0}' on page {1}", wordPattern.Pattern, page + 1); writer.WriteLine("Processing pattern search for '{0}' on page {1}", wordPattern.Pattern, page + 1); // parse all words var words = ReadWordsAgilityPack(documentNode, adminUser); foreach (var word in words) { if (wordPattern.IsMatchLastWord) { Log.Information("The current pattern matches the last-word: {0} = {1}. Current word: {2}", wordPattern.Pattern, wordPattern.LastWord, word.Value); hasFoundPattern = true; var wordRemoveDiacriticsToNorwegian = word.Value.RemoveDiacriticsToNorwegian(); // we might have had to add question marks at the end of the string to fix the length bug at the site if (wordRemoveDiacriticsToNorwegian == wordPattern.LastWord.TrimEnd('?')) { Log.Information("The current word matches the last-word: {0} = {1}", word.Value, wordPattern.LastWord); hasFoundLastWord = true; } } else { if (!hasFoundLastWord && hasFoundPattern) { // if the pattern not any longer match, we never found the word - has it been deleted? Log.Error("Warning! The current pattern does not any longer match the last-word: {0} = {1}. Current word: {2}", wordPattern.Pattern, wordPattern.LastWord, word.Value); writer.WriteLine("Warning! The current pattern does not any longer match the last-word: {0} = {1}. Current word: {2}", wordPattern.Pattern, wordPattern.LastWord, word.Value); hasMissedLastWord = true; return; } } if (hasFoundLastWord) { string currentValue = word.Value; // check if this is one of the buggy words from their site where the words found don't have the same length as the pattern says it should have if (wordPattern.Length != word.Value.Length) { Log.Error("Warning! The current word doesn't match the length of the query pattern: {0} = {1}", word.Value, wordPattern.Pattern); writer.WriteLine("Warning! The current word doesn't match the length of the query pattern: {0} = {1}", word.Value, wordPattern.Pattern); if (wordPattern.Length > word.Value.Length) { currentValue = currentValue + new string('?', wordPattern.Length - word.Value.Length); } else { currentValue = currentValue.Substring(0, wordPattern.Length); } } else { // everything is OK } // update that we are processing this word, ignore length and comment WordDatabaseService.UpdateState(db, source, new Word() { Value = currentValue.ToUpper(), Source = source, CreatedDate = DateTime.Now }, writer); GetWordSynonyms(word, driver, db, adminUser); } } // go to next page if exist var(hasFoundNextPage, pageNumber, pageUrl, pageNode) = NavigateToNextPageIfExist(driver, documentNode); if (hasFoundNextPage) { url = pageUrl; page = pageNumber; documentNode = pageNode; } else { break; } } }
// recursive function that progressively increases the depth if we get a too high word count // but maximum 3 permutations (abc???) private void ReadWordsByWordPermutationsRecursive(IWebDriver driver, WordPattern wordPattern, WordHintDbContext db, User adminUser) { // get word count var(wordCount, node, url, page) = GetWordCountByPattern(driver, wordPattern); if (wordCount == 0) { return; } else { Log.Information("Found {0} words when searching for '{1}' on page {2}", wordCount, wordPattern.Pattern, page + 1); writer.WriteLine("Found {0} words when searching for '{1}' on page {2}", wordCount, wordPattern.Pattern, page + 1); } // if we get too many words back, try to increase the pattern depth // but maximum 4 levels if (wordCount <= 108 || wordPattern.Depth > 3) { if (wordPattern.Depth > 3) { Log.Error("Warning! Pattern search depth is now {0}. Found {1} words when searching for '{2}' on page {3}", wordPattern.Depth, wordCount, wordPattern.Pattern, page + 1); } // process each word found using the specified word pattern ProcessWordsUntilEmpty(wordPattern, driver, db, adminUser, page, node, url); } else { // increment pattern var childPatterns = wordPattern.GetWordPatternChildren(); // recursively process children patterns foreach (var childPattern in childPatterns) { if (hasMissedLastWord) { return; } // if we have a last word - make sure to skip until the pattern matches if (!hasFoundLastWord && childPattern.LastWord != null && !childPattern.IsMatchLastWord) { // skip pattern Log.Information("Skipping pattern '{0}'.", childPattern.Pattern); } else { Log.Information("Processing pattern '{0}'.", childPattern.Pattern); ReadWordsByWordPermutationsRecursive(driver, childPattern, db, adminUser); } } } }
private void ReadWordsByWordPermutations(int letterLength, IWebDriver driver, WordHintDbContext db, User adminUser, string lastWord) { int wordLength = letterLength; int depth = 1; // if we are using only one letter, set the depth to zero to force the pattern to become ? if (letterLength == 1) { depth = 0; } // set wordlength based on last word if we are not processing in swarm mode and are continuing // to read patterns with increasing length // this assumes that we found the last word for the first pattern match if (!hasFoundLastWord && lastWord != null) { wordLength = lastWord.Length; // disabling this as it doesn't give any real benefits - only misses words that start with one character for high letter counts // if the word is longer than 3 characters, use the two first letters as pattern // if (wordLength > 2) // { // depth = 2; // } } var permutations = GetPermutations(depth); bool hasFound = false; foreach (var permutation in permutations) { if (hasMissedLastWord) { return; } // skip until we reach last word beginning if (!hasFoundLastWord && lastWord != null) { if (lastWord.ToLowerInvariant().StartsWith(permutation)) { hasFound = true; } } else { hasFound = true; } if (hasFound) { ReadWordsByWordPermutationsRecursive(driver, new WordPattern(permutation, wordLength, depth, lastWord), db, adminUser); } } }
private void ReadDescriptionsIntoDatabase(WordHintDbContext db, List <string> words) { #if DEBUG // Create new stopwatch. Stopwatch stopwatch = new Stopwatch(); // Begin timing. stopwatch.Start(); #endif Random rnd = new Random(); // find out which words have already a description var newWords = words.Where(value => !_description.Any(entry => entry.Key == value)); // find the words in the database var existingWords = db.Words .Include(w => w.RelatedFrom) .ThenInclude(w => w.WordTo) .Include(w => w.RelatedTo) .ThenInclude(w => w.WordFrom) .Where(x => newWords.Contains(x.Value)) .AsNoTracking(); foreach (var word in existingWords) { var wordText = word.Value; if (word.RelatedFrom.Count > 0) { // string hintText = word.RelatedFrom.Last().WordTo.Value // randomize the word descriptions int indexFrom = rnd.Next(0, word.RelatedFrom.Count - 1); string hintText = word.RelatedFrom.ToArray()[indexFrom].WordTo.Value; AddDescription(wordText, hintText); } else if (word.RelatedTo.Count > 0) { // string hintText = word.RelatedTo.Last().WordFrom.Value; // randomize the word descriptions int indexTo = rnd.Next(0, word.RelatedTo.Count - 1); string hintText = word.RelatedTo.ToArray()[indexTo].WordFrom.Value; AddDescription(wordText, hintText); } } #if DEBUG // Stop timing. stopwatch.Stop(); // Write result. if (_logger != null) { _logger.LogDebug("ReadDescriptionsIntoDatabase - Time elapsed: {0}", stopwatch.Elapsed); } else { Console.WriteLine("ReadDescriptionsIntoDatabase - Time elapsed: {0}", stopwatch.Elapsed); } #endif }
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IHostingEnvironment env, WordHintDbContext db , VersionedODataModelBuilder modelBuilder , IApiVersionDescriptionProvider provider ) { app.UseForwardedHeaders(); if (env.IsDevelopment()) { app.UseRequestResponseLogging(); app.UseDeveloperExceptionPage(); app.UseDatabaseErrorPage(); } else { app.UseExceptionHandler("/Error"); // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. app.UseHsts(); } // app.UseHttpsRedirection(); // disable to use within docker behind a proxy app.UseStaticFiles(); app.UseCookiePolicy(); // You would either call EnsureCreated() or Migrate(). // EnsureCreated() is an alternative that completely skips the migrations pipeline and just creates a database that matches you current model. // It's good for unit testing or very early prototyping, when you are happy just to delete and re-create the database when the model changes. // db.Database.EnsureDeleted(); // db.Database.EnsureCreated(); // Note! Therefore don't use EnsureDeleted() and EnsureCreated() but Migrate(); db.Database.Migrate(); app.UseCors("Everything"); app.UseAuthentication(); // add signalr hub url app.UseSignalR(routes => { routes.MapHub <CrossWordSignalRHub>("/crosswordsignalrhub"); }); // Add support for OData to MVC pipeline var models = modelBuilder.GetEdmModels(); // versioning API // var model = WordModelConfiguration.GetEdmModel(new ODataConventionModelBuilder()); // single odata API app.UseMvc(routes => { // routes.MapRoute( // name: "default", // template: "{controller=Home}/{action=Index}/{id?}"); // setup odata filters routes.Select().Expand().Filter().OrderBy().MaxTop(300).Count(); // version with query parameter routes.MapVersionedODataRoutes( routeName: "odata", routePrefix: "odata", models: models); // version by path // routes.MapVersionedODataRoutes( // routeName: "odata-bypath", // routePrefix: "odata/v{version:apiVersion}", // models: models); // setup non-versioned odata route // routes.MapODataServiceRoute("odata", "odata", model); // Error: Cannot find the services container for the non-OData route. // This can occur when using OData components on the non-OData route and is usually a configuration issue. // Call EnableDependencyInjection() to enable OData components on non-OData routes. // This may also occur when a request was mistakenly handled by the ASP.NET Core routing layer instead of the OData routing layer, // for instance the URL does not include the OData route prefix configured via a call to MapODataServiceRoute(). // Workaround: https://github.com/OData/WebApi/issues/1175 // routes.EnableDependencyInjection(); }); // Enable middleware to serve swagger-ui (HTML, JS, CSS, etc.), app.UseODataVersioningSwaggerDocumentation(modelBuilder, provider); // app.UseSwaggerDocumentation(); }
private void ReadWordsByWordUrl(string wordPrefix, string url, IWebDriver driver, WordHintDbContext db, User adminUser, string lastWord) { // go to word page try { driver.Navigate().GoToUrl(url); } catch (System.Exception) { // Log.Error("Timeout navigating to '{0}'", url); writer.WriteLine("Timeout navigating to '{0}'", url); return; } Log.Information("Processing word search for '{0}'", wordPrefix); writer.WriteLine("Processing word search for '{0}'", wordPrefix); // read the whole document into a HtmlNode HtmlNode doc = driver.GetDocumentNode(); // and parse using agility pack var words = ParseWordsAgilityPack(doc, adminUser); bool doSkip = true; foreach (var wordAndHref in words) { var word = wordAndHref.Item1; var href = wordAndHref.Item2; var wordText = word.Value; // skip until we get to the last word if (doSkip && lastWord != null && lastWord != wordText) { Log.Information("Skipping alphabetic word '{0}' until we find '{1}'", wordText, lastWord); writer.WriteLine("Skipping alphabetic word '{0}' until we find '{1}'", wordText, lastWord); continue; } doSkip = false; // make sure we don't skip on the next word after we have skipped // update that we are processing this word WordDatabaseService.UpdateState(db, source, new Word() { Value = wordText, Comment = wordPrefix, CreatedDate = DateTime.Now }, writer, true); GetWordSynonyms(word, driver, db, adminUser, href); } }
private void ReadWordsFromUrl(WordHintDbContext db, User adminUser, string lastWord) { using (WebClient client = new WebClient()) using (Stream stream = client.OpenRead(JSON_URL)) using (StreamReader streamReader = new StreamReader(stream)) using (JsonTextReader reader = new JsonTextReader(streamReader)) { reader.SupportMultipleContent = true; string currentValue = null; List <string> currentList = null; int totalCount = 25000; int count = 0; bool hasFound = false; var serializer = new JsonSerializer(); while (reader.Read()) { // output the stream one chunk at a time // Log.Information(string.Format("{0,-12} {1}", // reader.TokenType.ToString(), // reader.Value != null ? reader.Value.ToString() : "(null)")); switch (reader.TokenType) { // JsonToken.StartObject = deserialize only when there's "{" character in the stream case JsonToken.StartObject: break; // JsonToken.PropertyName = deserialize only when there's a "text": in the stream case JsonToken.PropertyName: currentValue = reader.Value.ToString(); break; // JsonToken.String = deserialize only when there's a "text" in the stream case JsonToken.String: currentList.Add(reader.Value.ToString()); break; // JsonToken.StartArray = deserialize only when there's "[" character in the stream case JsonToken.StartArray: currentList = new List <string>(); break; // JsonToken.EndArray = deserialize only when there's "]" character in the stream case JsonToken.EndArray: count++; // skip until we reach last word beginning if (lastWord != null) { if (currentValue.ToUpperInvariant().Equals(lastWord)) { hasFound = true; } } else { hasFound = true; } // store to database if (hasFound) { // update that we are processing this word, ignore length and comment WordDatabaseService.UpdateState(db, source, new Word() { Value = currentValue.ToUpper(), Source = source, CreatedDate = DateTime.Now }, writer, true); // disable storing state since we are doing it manually above WordDatabaseService.AddToDatabase(db, source, adminUser, currentValue, currentList, writer, false); // if (writer != null) writer.WriteLine("Added '{0} => {1}'", currentValue, string.Join(",", currentList)); if ((count % 10) == 0) { if (writer != null) { writer.WriteLine("[{0}] / [{1}]", count, totalCount); } } } // and reset currentList = null; currentValue = null; break; // JsonToken.EndObject = deserialize only when there's "}" character in the stream case JsonToken.EndObject: currentList = null; currentValue = null; break; } } } /* * // reading the whole thing took approx the same time as the streaming version * { * var json = streamReader.ReadToEnd(); * var jobj = JObject.Parse(json); * * var totalCount = jobj.Properties().Count(); * int count = 0; * foreach (var item in jobj.Properties()) * { * count++; * * var currentValue = item.Name; * var currentList = item.Values().Select(a => a.Value<string>()); * * WordDatabaseService.AddToDatabase(db, source, adminUser, currentValue, currentList); * * // if (writer != null) writer.WriteLine("Added '{0} => {1}'", currentValue, string.Join(",", currentList)); * if (writer != null) writer.WriteLine("[{0}] / [{1}]", count, totalCount); * } * } */ }