Example #1
0
        /// <summary>
        /// Main algorithm to return author searches from the database. We start with prefering the whole string matches
        /// and then starts to include more and more granular results, by words count match
        /// /// </summary>
        /// <param name="db">Database context</param>
        /// <param name="query">Current query</param>
        /// <returns>List of returned books</returns>
        public static IEnumerable <Author> SearchAuthor(BookRecommenderContext db, string query)
        {
            // First try to find the exact query within books
            var startsWithFound = db.Authors.Where(a => a.NameEn.ToLower().StartsWith(query.ToLower()));
            var containsFound   = db.Authors.Where(a => a.NameEn.ToLower().Contains(query.ToLower()));
            var wholeQueryFound = startsWithFound.Concat(containsFound);

            var splitQuery = query.ToLower().Split(new char[] { ' ' });

            // if query is only one word - return
            if (splitQuery.Length == 1)
            {
                return(wholeQueryFound);
            }

            // then find books which have even only single query word, and order them by the amounts of query words found
            var countDictionary = new Dictionary <Author, int>();

            foreach (var queryWord in splitQuery)
            {
                var authors = db.Authors.Where(a => a.NameEn.ToLower().Contains(queryWord));
                foreach (var author in authors)
                {
                    int value;
                    countDictionary[author] = countDictionary.TryGetValue(author, out value) ? ++value : 1;
                }
            }
            var orderedResult = countDictionary.OrderByDescending(b => b.Value).Select(b => b.Key);


            //combine the whole query result with single words query result in way that whole query is preffered
            var combQuery = wholeQueryFound.Concat(orderedResult).Distinct();

            return(combQuery);
        }
        /// <summary>
        /// Method which takes IEnumerable with the data and calls the assigned line action on every line of the data.
        /// The data needs to fit inside the line action.
        /// This method is meant to be used when mining from command line
        /// </summary>
        /// <param name="data">Enumerable with the data lines</param>
        /// <param name="lineAction">Action which will be executed on each line</param>
        protected void UpdateDatabase <T>(IEnumerable <T> data, LineAction <T> lineAction)
        {
            // Writeout the delegate name
            System.Console.WriteLine(lineAction.GetMethodInfo().Name);

            bool successfull = false;

            do
            {
                var numberOfTries = 1;
                try
                {
                    //Execute query -- retrieve collection only once
                    System.Console.WriteLine("Querying Endpoind");
                    var listData = data.ToList();
                    System.Console.WriteLine("Updating database");
                    using (var db = new BookRecommenderContext())
                    {
                        //Create new console progress counter
                        using (var counter = new Counter(listData.Count))
                        {
                            // Insert all books in database
                            foreach (var line in listData)
                            {
                                lineAction(line, db);
                                counter.Update();
                            }
                        }
                        System.Console.WriteLine("Saving database");
                        db.SaveChanges();
                        successfull = true;
                    }
                }
                catch (Exception ex)
                {
                    // If something went wrong, wait 10 sec and then try again
                    numberOfTries++;

                    System.Console.WriteLine(ex.ToString());
                    System.Console.WriteLine("Try again, attempt number " + numberOfTries);
                }

                // If something went wrong, wait 10 sec and then try again
                if (!successfull)
                {
                    System.Threading.Tasks.Task.Delay(10000).Wait();
                }
            } while (!successfull);
        }
Example #3
0
        public void Initialize(BookRecommenderContext db)
        {
            var sw = Stopwatch.StartNew();

            InitializeAuthors(db);
            InitializeGenres(db);
            InitializeCharacters(db);
            InitializeTags(db);

            InitializeNames(db);

            sw.Stop();
            var elapsed = sw.ElapsedMilliseconds;

            System.Console.WriteLine($"Spreading cache entries: {SimilarBooksByAuthor.Keys.Count}#");
            System.Console.WriteLine($"Spreading cache init took: {elapsed}ms");
        }
Example #4
0
        /// <summary>
        /// Simpler and faster version of search to be used in dynamically loaded whispering when searching.
        /// </summary>
        /// <param name="db">Database context</param>
        /// <param name="query">Current query</param>
        /// <param name="howManyTop">how many to show</param>
        /// <returns>List of recommendations</returns>
        public static List <string> Autocomplete(BookRecommenderContext db, string query, int howManyTop)
        {
            if (string.IsNullOrEmpty(query))
            {
                return(new List <string>());
            }
            var booksStartingWith   = db.Books.Select(b => b.NameEn).Where(b => b.ToLower().StartsWith(query.ToLower())).Take(howManyTop).ToList();
            var authorsStartingWith = db.Authors.Select(a => a.NameEn).Where(a => a.ToLower().StartsWith(query.ToLower())).Take(howManyTop).ToList();
            var startsWith          = booksStartingWith.Concat(authorsStartingWith);

            var booksContaining   = db.Books.Select(b => b.NameEn).Where(b => b.ToLower().Contains(query.ToLower())).Take(howManyTop).ToList();
            var authorsContaining = db.Authors.Select(a => a.NameEn).Where(a => a.ToLower().Contains(query.ToLower())).Take(howManyTop).ToList();
            var contains          = booksContaining.Concat(authorsContaining);

            var final = startsWith.Concat(contains).Distinct().Take(howManyTop);

            return(final.ToList());
        }
Example #5
0
        // public void InitializeGenres(BookRecommenderContext db)
        // {
        //     // Book to genres
        //     var bookGenreList = db.BooksGenres.Select(t => new { t.BookId, t.GenreId })
        //         .ToList();

        //     var bookToGenres = bookGenreList.GroupBy(bg => bg.BookId)
        //                     .ToDictionary(g => g.Key, g => g.Select(bg => bg.GenreId).ToList());

        //     var genresToBooks = bookGenreList.GroupBy(bg => bg.GenreId)
        //         .ToDictionary(g => g.Key, g => g.Select(bg => bg.BookId).ToList());

        //     SimilarBooksByGenre = bookToGenres
        //         .ToDictionary(
        //             kvp => kvp.Key,
        //             kvp => kvp.Value
        //                 .SelectMany(g => genresToBooks[g])
        //                 .ToList());

        //     var total = SimilarBooksByGenre.SelectMany(f => f.Value).Count();
        //     var aproxMemMB = total * 4d / 1024 / 1024;
        //     System.Console.WriteLine($"Genres total hops available: {total}, MB: {aproxMemMB}");
        // }


        public void InitializeGenres(BookRecommenderContext db)
        {
            // Book to genres
            var bookGenreList = db.BooksGenres.Select(t => new { t.BookId, t.GenreId })
                                .ToList();

            BooksGenres = bookGenreList.GroupBy(bg => bg.BookId)
                          .ToDictionary(g => g.Key, g => g.Select(bg => bg.GenreId).ToList());

            GenresBooks = bookGenreList.GroupBy(bg => bg.GenreId)
                          .ToDictionary(g => g.Key, g => g.Select(bg => bg.BookId).ToList());


            var total = BooksGenres.SelectMany(f => f.Value).Count()
                        + GenresBooks.SelectMany(f => f.Value).Count();

            var aproxMemMB = total * 4d / 1024 / 1024;

            System.Console.WriteLine($"Genres total hops available(2 tables): {total}, MB: {aproxMemMB}");
        }
        /// <summary>
        /// Method which takes IEnumerable with the data and calls the assigned line action on every line of the data.
        /// The data needs to fit inside the line action.
        /// This method is meant to be used when mining from web interface
        /// </summary>
        /// <param name="data">Enumerable with the data lines</param>
        /// <param name="lineAction">Action which will be executed on each line</param>
        /// <param name="miningState">Mining state of the operation from the MiningProxySingleton used to monitor the data mining</param>
        protected void UpdateDatabase <T>(IEnumerable <T> data, LineAction <T> lineAction, MiningState miningState)
        {
            // fall back for deprecated commandline mining
            if (miningState == null)
            {
                UpdateDatabase <T>(data, lineAction);
                return;
            }

            try
            {
                // set the mining state
                miningState.CurrentState = MiningStateType.RunningQueryingEndpoint;
                var listData = data.ToList();
                miningState.CurrentState = MiningStateType.Running;
                var currentPosition = 0;
                using (var db = new BookRecommenderContext())
                {
                    // proccess each line using the line action and log changes to the mining state
                    foreach (var line in listData)
                    {
                        lineAction(line, db);
                        currentPosition++;
                        miningState.Message = String.Format("{0}/{1}",
                                                            currentPosition, listData.Count);
                    }
                    miningState.CurrentState = MiningStateType.RunningSavingToDatabase;
                    db.SaveChanges();
                    miningState.CurrentState = MiningStateType.Completed;
                    miningState.Message      = DateTime.Now.ToString();
                }
            }
            catch (Exception ex)
            {
                // If something went wrong, wait 10 sec and then try again
                miningState.CurrentState = MiningStateType.Error;
                miningState.Message      = ex.Message;
            }
        }
Example #7
0
        public void InitializeAuthors(BookRecommenderContext db)
        {
            // Book to authors
            var bookAuthorList = db.BooksAuthors.Select(t => new { t.BookId, t.AuthorId })
                                 .ToList();

            var bookToAuthors = bookAuthorList.GroupBy(ba => ba.BookId)
                                .ToDictionary(g => g.Key, g => g.Select(ba => ba.AuthorId).ToList());

            var authorToBooks = bookAuthorList.GroupBy(ba => ba.AuthorId)
                                .ToDictionary(g => g.Key, g => g.Select(ba => ba.BookId).ToList());

            SimilarBooksByAuthor = bookToAuthors
                                   .ToDictionary(
                kvp => kvp.Key,
                kvp => kvp.Value
                .SelectMany(a => authorToBooks[a])
                .ToList());

            var total      = SimilarBooksByAuthor.SelectMany(f => f.Value).Count();
            var aproxMemMB = total * 4d / 1024 / 1024;

            System.Console.WriteLine($"Authors total hops available: {total}, MB: {aproxMemMB}");
        }
Example #8
0
        public void InitializeTags(BookRecommenderContext db)
        {
            // Book to tags
            var bookTagList = db.Tags.Select(t => new { t.BookId, t.TagId })
                              .ToList();

            var bookToTags = bookTagList.GroupBy(bt => bt.BookId)
                             .ToDictionary(g => g.Key, g => g.Select(bt => bt.TagId).ToList());

            var tagsToBooks = bookTagList.GroupBy(bt => bt.TagId)
                              .ToDictionary(g => g.Key, g => g.Select(bt => bt.BookId).ToList());

            SimilarBooksByTags = bookToTags
                                 .ToDictionary(
                kvp => kvp.Key,
                kvp => kvp.Value
                .SelectMany(t => tagsToBooks[t])
                .ToList());

            var total      = SimilarBooksByTags.SelectMany(f => f.Value).Count();
            var aproxMemMB = total * 4d / 1024 / 1024;

            System.Console.WriteLine($"Tags total hops available: {total}, MB: {aproxMemMB}");
        }
Example #9
0
        public void InitializeCharacters(BookRecommenderContext db)
        {
            // Book to genres
            var bookCharacterList = db.BooksCharacters.Select(t => new { t.BookId, t.CharacterId })
                                    .ToList();

            var bookToCharacters = bookCharacterList.GroupBy(bc => bc.BookId)
                                   .ToDictionary(g => g.Key, g => g.Select(bc => bc.CharacterId).ToList());

            var charactersToBooks = bookCharacterList.GroupBy(bc => bc.CharacterId)
                                    .ToDictionary(g => g.Key, g => g.Select(bc => bc.BookId).ToList());

            SimilarBooksByCharacters = bookToCharacters
                                       .ToDictionary(
                kvp => kvp.Key,
                kvp => kvp.Value
                .SelectMany(g => charactersToBooks[g])
                .ToList());

            var total      = SimilarBooksByCharacters.SelectMany(f => f.Value).Count();
            var aproxMemMB = total * 4d / 1024 / 1024;

            System.Console.WriteLine($"Characters total hops available: {total}, MB: {aproxMemMB}");
        }
Example #10
0
        public void InitializeNames(BookRecommenderContext db)
        {
            var books = db.Books.ToList();

            BooksNames = books.ToDictionary(b => b.BookId, b => b.GetNameEn());
        }