Exemple #1
0
        public void CustomTokenizer_tokenize_orm()
        {
            using (var db = new SQLiteDatabase(":memory:"))
            {
                db.Configure(SQLiteDatabaseConfiguration.SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, true, 1);
                var tok = new StopWordTokenizer(db);
                db.SetTokenizer(tok);

                // insert data as in SQLite exemple
                db.Save(new Mail {
                    docid = 1, Subject = "software feedback", Body = "found it too slow"
                });
                db.Save(new Mail {
                    docid = 2, Subject = "software feedback", Body = "no feedback"
                });
                db.Save(new Mail {
                    docid = 3, Subject = "slow lunch order", Body = "was a software problem"
                });

                // check result
                Assert.AreEqual(3, db.LoadAll <Mail>().Count());

                Assert.AreEqual("1,2", string.Join(",", db.Load <Mail>("WHERE Subject MATCH 'software'").Select(m => m.docid)));
                Assert.AreEqual("2", string.Join(",", db.Load <Mail>("WHERE body MATCH 'feedback'").Select(m => m.docid)));
                Assert.AreEqual("1,2,3", string.Join(",", db.Load <Mail>("WHERE mail MATCH 'software'").Select(m => m.docid)));
                Assert.AreEqual("1,3", string.Join(",", db.Load <Mail>("WHERE mail MATCH 'slow'").Select(m => m.docid)));
                Assert.AreEqual("", string.Join(",", db.Load <Mail>("WHERE mail MATCH 'no'").Select(m => m.docid)));
            }
        }
Exemple #2
0
        public void CustomTokenizer_tokenize()
        {
            using (var db = new SQLiteDatabase(":memory:"))
            {
                using (var tok = db.GetTokenizer("unicode61", "remove_diacritics=0", "tokenchars=.=", "separators=X"))
                {
                    Assert.AreEqual("hellofriends", string.Join(string.Empty, tok.Tokenize("hello friends")));
                    GC.Collect(1000, GCCollectionMode.Forced, true);
                }

                var sp = new StopWordTokenizer(db);
                Assert.AreEqual(1, db.Configure(SQLiteDatabaseConfiguration.SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, true, 1));

                db.SetTokenizer(sp);
                db.ExecuteNonQuery("CREATE VIRTUAL TABLE tok1 USING fts3tokenize('" + sp.Name + "');");

                var tokens = db.LoadRows(@"SELECT token, start, end, position FROM tok1 WHERE input=?;",
                                         "This is a test sentence.");

                Assert.AreEqual("testsentence", string.Join(string.Empty, tokens.Select(t => t["token"])));
            }
        }