public Searcher(FullTextIndex index)
{
_index = index;
_tx = _index.StorageEnvironment.NewTransaction(TransactionFlags.Read);
_docs = _tx.ReadTree("Docs");
}
public void Initialize(FullTextIndex index, Transaction tx, IndexingConventions.ScorerCalc score)
{
Index = index;
Transaction = tx;
Score = score;
Init();
}
public Indexer(FullTextIndex parent)
{
_parent = parent;
_analyzer = _parent.Analyzer;
_bufferPool = _parent.BufferPool;
AutoFlush = true;
}
public void CanCreateAndDisposeIndexer()
{
using (var fti = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (fti.CreateIndexer())
{
}
}
}
public void CanDelete()
{
using (var fullTextIndex = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren Eini");
indexer.NewIndexEntry();
indexer.AddField("Name", "Ayende Rahien");
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
Assert.Equal(1, searcher.Query(new TermQuery("Name", "oren")).Count());
Assert.Equal(1, searcher.Query(new TermQuery("Name", "rahien")).Count());
}
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.DeleteIndexEntry(1);
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
Assert.Equal(0, searcher.Query(new TermQuery("Name", "oren")).Count());
Assert.Equal(1, searcher.Query(new TermQuery("Name", "rahien")).Count());
}
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.DeleteIndexEntry(2);
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
Assert.Equal(0, searcher.Query(new TermQuery("Name", "oren")).Count());
Assert.Equal(0, searcher.Query(new TermQuery("Name", "rahien")).Count());
}
}
}
public FullTextIndexTests()
{
this.index = new FullTextIndexBuilder <string>()
.WithItemTokenization <TestObject>(
o => o.WithKey(i => i.Id)
.WithField("Text1", i => i.Text1, opts => opts.CaseInsensitive(false))
.WithField("Text2", i => i.Text2)
.WithField("Text3", i => i.Text3, opts => opts.WithStemming()))
.WithItemTokenization <TestObject2>(
o => o.WithKey(i => i.Id)
.WithField("MultiText", i => i.Text))
.WithItemTokenization <TestObject3>(
o => o.WithKey(i => i.Id)
.WithField("TextAsync", i => Task.Run(() => i.Text))
.WithField("MultiTextAsync", i => Task.Run(() => (IEnumerable <string>)i.MultiText)))
.Build();
}
public void CanDeleteAndCompact()
{
using (var fullTextIndex = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
fullTextIndex.Conventions.AutoCompact = false;
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.Index("Name", "Oren Eini");
indexer.NewIndexEntry();
indexer.Index("Name", "Ayende Rahien");
indexer.Flush();
}
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.DeleteIndexEntry(1);
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
Assert.Equal(0, searcher.Query(new TermQuery("Name", "oren")).Count());
Assert.Equal(1, searcher.Query(new TermQuery("Name", "rahien")).Count());
}
Assert.Equal(2, fullTextIndex.NumberOfDocuments);
Assert.Equal(1, fullTextIndex.NumberOfDeletes);
fullTextIndex.RunCompaction();
using (var searcher = fullTextIndex.CreateSearcher())
{
Assert.Equal(0, searcher.Query(new TermQuery("Name", "oren")).Count());
Assert.Equal(1, searcher.Query(new TermQuery("Name", "rahien")).Count());
}
Assert.Equal(1, fullTextIndex.NumberOfDocuments);
Assert.Equal(0, fullTextIndex.NumberOfDeletes);
}
}
/// <summary>
/// Initialize an empty datastore from a type description
/// </summary>
/// <param name="collectionSchema"></param>
/// <param name="evictionPolicy"></param>
/// <param name="fullTextConfig"></param>
public DataStore(CollectionSchema collectionSchema, EvictionPolicy evictionPolicy,
FullTextConfig fullTextConfig)
{
_fullTextConfig = fullTextConfig;
CollectionSchema = collectionSchema ?? throw new ArgumentNullException(nameof(collectionSchema));
EvictionPolicy = evictionPolicy ?? throw new ArgumentNullException(nameof(evictionPolicy));
//initialize the primary key dictionary
DataByPrimaryKey = new Dictionary <KeyValue, PackedObject>();
//initialize the unique keys dictionaries (one by unique key)
_dataByUniqueKey = new Dictionary <string, Dictionary <KeyValue, PackedObject> >();
foreach (var keyInfo in collectionSchema.UniqueKeyFields)
{
_dataByUniqueKey.Add(keyInfo.Name, new Dictionary <KeyValue, PackedObject>());
}
//initialize the indexes (one by index key)
_dataByIndexKey = new Dictionary <string, IndexBase>();
// scalar indexed fields
foreach (var indexField in collectionSchema.IndexFields)
{
var index = IndexFactory.CreateIndex(indexField);
_dataByIndexKey.Add(indexField.Name, index);
}
// create the full-text index if required
if (collectionSchema.FullText.Count > 0)
{
_fullTextIndex = new FullTextIndex(fullTextConfig)
{
// a function that allows the full text engine to find the original line of text
LineProvider = pointer => DataByPrimaryKey[pointer.PrimaryKey].TokenizedFullText[pointer.Line]
}
}
;
}
public void WillFilterStopWords()
{
using (var fti = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fti.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren and Ayende");
indexer.Flush();
}
using (var searcher = fti.CreateSearcher())
{
Assert.Empty(searcher.Query(new TermQuery("Name", "and")));
}
}
}
public void CanQueryUsingMissingField()
{
using (var fti = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fti.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren Eini");
indexer.Flush();
}
using (var searcher = fti.CreateSearcher())
{
Assert.Empty(searcher.Query(new TermQuery("Foo", "Arava")));
}
}
}
protected virtual async Task ExtractFulltextIndexesAsync(ExtractionContext context, CancellationToken token)
{
var query = BuildExtractFullTextIndexesQuery(context);
var currentTableId = 0;
ColumnResolver table = null;
FullTextIndex index = null;
var cmd = Connection.CreateCommand(query);
await using (cmd.ConfigureAwait(false)) {
var reader = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false);
await using (reader.ConfigureAwait(false)) {
while (await reader.ReadAsync(token).ConfigureAwait(false))
{
ReadFullTextIndexColumnData(reader, context, ref currentTableId, ref table, ref index);
}
}
}
}
public async Task ReadIntoAsync(FullTextIndex <TKey> index)
{
await this.FillBufferAsync().ConfigureAwait(false);
var itemCount = this.reader.ReadInt32();
for (var i = 0; i < itemCount; i++)
{
var id = this.reader.ReadInt32();
var key = this.keySerializer.Read(this.reader);
index.IdPool.Add(id, key);
}
index.SetRootWithLock(this.DeserializeNode(index.IndexNodeFactory, 0));
if (this.reader.ReadInt32() != -1)
{
throw new DeserializationException(ExceptionMessages.MissingIndexTerminator);
}
}
private static void Main(string[] args)
{
// Create a full text index with default settings
var index = new FullTextIndex <string>();
// Index
index.Index("A", "This is some text associated with A: fizz");
index.Index("B", "Some buzz text for B");
index.Index("C", "Text associated with C is both fizz and buzz");
// Search for text containing both Fizz *and* Buzz
var results = index.Search("Fizz Buzz").ToList();
// Output: Items with both Fizz and Buzz: 1
Console.WriteLine($"Items with both Fizz and Buzz: {results.Count}");
// Search for text containing both Fizz *or* Buzz
results = index.Search("Fizz | Buzz").ToList();
// Outputs: Items with Fizz or Buzz: 3
Console.WriteLine($"Items with Fizz or Buzz: {results.Count}");
}
public void CanQueryAndUpdate()
{
using (var fullTextIndex = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.NewIndexEntry(); // doc 1
Assert.Equal(1L, indexer.CurrentDocumentId);
indexer.AddField("Name", "Oren Eini");
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
Assert.Equal(1, searcher.Query(new TermQuery("Name", "oren")).Count());
}
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.UpdateIndexEntry(1); // doc 1
Assert.Equal(1L, indexer.CurrentDocumentId);
indexer.AddField("Name", "Ayende Rahien");
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
Assert.Equal(0, searcher.Query(new TermQuery("Name", "oren")).Count());
Assert.Equal(1, searcher.Query(new TermQuery("Name", "ayende")).Count());
}
}
}
public void CanDoPhraseQuery()
{
using (var fullTextIndex = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("PhraseText", "Not a match.", options: FieldOptions.TermPositions);
indexer.NewIndexEntry();
indexer.AddField("PhraseText", "RavenDB is a very cool database to work with.", options: FieldOptions.TermPositions);
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
var results = searcher.QueryTop(new PhraseQuery("PhraseText", "ravendb", "cool", "database")
{
Slop = 4
}, 5);
Assert.Equal(1, results.Results.Length);
Assert.Equal(2, results.Results[0].DocumentId);
}
using (var searcher = fullTextIndex.CreateSearcher())
{
var results = searcher.QueryTop(new PhraseQuery("PhraseText", "database", "cool", "ravendb")
{
Slop = 4
}, 5);
Assert.Equal(0, results.Results.Length);
}
}
}
public async Task <IEnumerable <FullTextIndex> > SelectIndexesAsync()
{
IList <FullTextIndex> output = null;
using (var context = _dbContext)
{
output = await context.ExecuteReaderAsync(
CommandType.Text,
BySql,
async reader =>
{
if ((reader != null) && (reader.HasRows))
{
output = new List <FullTextIndex>();
while (await reader.ReadAsync())
{
var index = new FullTextIndex();
index.PopulateModel(reader);
output.Add(index);
}
}
return(output);
});
// Remove table prefix from table names
if (output != null)
{
foreach (var index in output)
{
index.TableName = index.TableName.Replace(context.Configuration.TablePrefix, "");
}
}
}
return(output);
}
/// <inheritdoc/>
protected override IPathNode VisitFullTextIndex(FullTextIndex index)
{
var tableInfo = currentTable;
var name = index.Name.IsNullOrEmpty()
? string.Format("FT_{0}", index.DataTable.Name)
: index.Name;
var ftIndex = new StorageFullTextIndexInfo(tableInfo, name)
{
FullTextCatalog = index.FullTextCatalog,
ChangeTrackingMode = ConvertChangeTrackingMode(index.ChangeTrackingMode)
};
foreach (var column in index.Columns)
{
var columnInfo = tableInfo.Columns[column.Column.Name];
string typeColumn = null;
if (column.TypeColumn != null)
{
typeColumn = tableInfo.Columns[column.TypeColumn.Name].Name;
}
new FullTextColumnRef(ftIndex, columnInfo, column.Languages.Single().Name, typeColumn);
}
return(ftIndex);
}
internal protected string GetFulltextVector(SqlCompilerContext context, FullTextIndex index)
{
var sb = new StringBuilder();
sb.Append("(");
var languageGroups = index
.Columns
.SelectMany(column => column.Languages, (column, language) => new { column, language })
.GroupBy(pair => pair.language, pair => pair.column)
.ToList();
for (int i = 0; i < languageGroups.Count; i++)
{
if (i != 0)
{
sb.Append(" || ");
}
var group = languageGroups[i];
var columns = group.ToList();
sb.Append("to_tsvector('");
sb.Append(group.Key.Name);
sb.Append("'::regconfig, ");
for (int j = 0; j < columns.Count; j++)
{
if (j != 0)
{
sb.Append(" || ' '::text");
}
IndexColumn column = columns[j];
sb.AppendFormat("({0})::text", QuoteIdentifier(column.Name));
}
sb.Append(")");
}
sb.Append(")");
return(sb.ToString());
}
internal protected string GetFulltextVector(SqlCompilerContext context, FullTextIndex index)
{
var sb = new StringBuilder("(");
var languageGroups = index
.Columns
.SelectMany(column => column.Languages, (column, language) => new { column, language })
.GroupBy(pair => pair.language, pair => pair.column);
var isFirstOuter = true;
foreach (var languageGroup in languageGroups)
{
if (!isFirstOuter)
{
_ = sb.Append(" || ");
}
isFirstOuter = false;
var columns = languageGroup.ToList();
var isFirstInner = true;
_ = sb.Append("to_tsvector('")
.Append(languageGroup.Key.Name)
.Append("'::regconfig, ");
foreach (var language in languageGroup)
{
if (!isFirstInner)
{
_ = sb.Append(" || ' '::text");
}
isFirstInner = false;
_ = sb.AppendFormat("({0})::text", QuoteIdentifier(language.Name));
}
_ = sb.Append(")");
}
return(sb.Append(")").ToString());
}
public IndexNavigatorTests()
{
this.index = new FullTextIndex <string>();
this.index.Index("A", "Triumphant elephant strode elegantly with indifference to shouting subjects, giving withering looks to individuals");
this.sut = new IndexNavigator(this.index.Root);
}
private void ReadFullTextIndexColumnData(DbDataReader reader, ExtractionContext context,
ref int currentTableId, ref ColumnResolver table, ref FullTextIndex index)
{
var nextTableId = reader.GetInt32(1);
if (currentTableId != nextTableId)
{
GetDataTable(nextTableId, context, ref currentTableId, ref table);
index = table.Table.CreateFullTextIndex(string.Empty);
index.FullTextCatalog = reader.GetBoolean(4)
? null
: reader.GetString(3);
index.UnderlyingUniqueIndex = reader.GetString(8);
index.ChangeTrackingMode = GetChangeTrackingMode(reader.GetString(9), reader.IsDBNull(10));
}
var column = index.CreateIndexColumn(table.GetColumn(reader.GetInt32(5)));
column.TypeColumn = (reader.IsDBNull(6)) ? null : table.GetColumn(reader.GetInt32(6));
column.Languages.Add(new Language(reader.GetString(7)));
}
public void WillFilterStopWords()
{
using (var fti = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fti.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren and Ayende");
indexer.Flush();
}
using (var searcher = fti.CreateSearcher())
{
Assert.Empty(searcher.Query(new TermQuery("Name", "and")));
}
}
}
public override void Visit(SqlFreeTextTable node)
{
if (node.TargetColumns.Count != 1 || node.TargetColumns[0] != node.TargetTable.Asterisk)
{
throw new NotSupportedException(Strings.ExFreeTextSearchOnCustomColumnsNotSupported);
}
FullTextIndex fullTextIndex = node.TargetTable.DataTable.Indexes.OfType <FullTextIndex>().Single();
string alias = context.TableNameProvider.GetName(node);
string vector = ((Translator)translator).GetFulltextVector(context, fullTextIndex);
string tableName = translator.QuoteIdentifier(node.TargetTable.Name);
int internalColumnIndex = 0;
while (node.Columns["column" + internalColumnIndex] != null)
{
internalColumnIndex++;
}
string vectorName = translator.QuoteIdentifier("column" + internalColumnIndex);
internalColumnIndex++;
while (node.Columns["column" + internalColumnIndex] != null)
{
internalColumnIndex++;
}
string queryName = translator.QuoteIdentifier("column" + internalColumnIndex);
context.Output.AppendText("(SELECT ");
for (int columnIndex = 0; columnIndex < node.Columns.Count - 1; columnIndex++)
{
if (columnIndex != 0)
{
context.Output.AppendText(translator.ColumnDelimiter);
}
context.Output.AppendText(translator.QuoteIdentifier(node.Columns[columnIndex].Name));
}
context.Output.AppendText(translator.ColumnDelimiter);
context.Output.AppendText("ts_rank_cd(");
context.Output.AppendText(vectorName);
context.Output.AppendText(translator.ArgumentDelimiter);
context.Output.AppendText(queryName);
context.Output.AppendText(") AS");
context.Output.AppendText(translator.QuoteIdentifier(node.Columns[node.Columns.Count - 1].Name));
context.Output.AppendText(" FROM (SELECT ");
for (int columnIndex = 0; columnIndex < node.Columns.Count - 1; columnIndex++)
{
if (columnIndex != 0)
{
context.Output.AppendText(translator.ColumnDelimiter);
}
context.Output.AppendText(translator.QuoteIdentifier(node.Columns[columnIndex].Name));
}
context.Output.AppendText(translator.ColumnDelimiter);
context.Output.AppendText(string.Format("{0} AS {1}", vector, vectorName));
context.Output.AppendText(translator.ColumnDelimiter);
var languages = fullTextIndex
.Columns
.SelectMany(column => column.Languages)
.Select(language => language.Name)
.Distinct()
.ToList();
context.Output.AppendText("(");
for (int i = 0; i < languages.Count; i++)
{
if (i != 0)
{
context.Output.AppendText(" || ");
}
var language = languages[i];
context.Output.AppendText("to_tsquery('");
context.Output.AppendText(language);
context.Output.AppendText("'::regconfig, ");
context.Output.AppendText("replace(trim(regexp_replace(");
node.FreeText.AcceptVisitor(this);
context.Output.AppendText(@",'\\W+', ' ', 'g')),' ', '|')");
context.Output.AppendText(")");
}
context.Output.AppendText(")");
context.Output.AppendText(string.Format(" AS {0}", queryName));
context.Output.AppendText(string.Format(" FROM {0}) AS {1} WHERE {2} @@ {3})", tableName, alias, vectorName, queryName));
}
public FullTextIndexTests()
{
index = new FullTextIndex(1.Seconds());
}
/// <summary>
/// This method will create a full-text index on a table with an associated stoplist
/// </summary>
public static void CreateFullTextIndex()
{
Console.WriteLine("Running CreateFullTextIndex Sample...");
try
{
Server server = new Server(".");
server.ConnectionContext.Connect();
Console.WriteLine("Connected to '{0}' server", server.Name);
Database db = new Database(server, "iFTSSampleDB2");
db.Create();
Console.WriteLine("Database '{0}' created", db.Name);
// Create a default full-text catalog for the database
FullTextCatalog ftcat = new FullTextCatalog(db, "ftcatalog");
ftcat.IsDefault = true;
ftcat.Create();
Console.WriteLine("FullTextCatalog '{0}' created", ftcat.Name);
// Add 2 columns to the table
Table tab = new Table(db, "tab");
tab.Columns.Add(new Column(tab, "col_fti", DataType.NVarChar(1000)));
Column col = new Column(tab, "col_unique", DataType.Int);
col.Nullable = false;
tab.Columns.Add(col);
// Create a table with a unique index on it
Index idxUnique = new Index(tab, "tab_unique_idx");
idxUnique.IndexKeyType = IndexKeyType.DriUniqueKey;
idxUnique.IndexedColumns.Add(new IndexedColumn(idxUnique, col.Name));
tab.Indexes.Add(idxUnique);
tab.Create();
Console.WriteLine("Table '{0}' created", tab.Name);
// Add a column to the full-text index and associate it with a unique index
FullTextIndex fti = new FullTextIndex(tab);
fti.IndexedColumns.Add(new FullTextIndexColumn(fti, "col_fti"));
fti.UniqueIndexName = idxUnique.Name;
string stopword = "goodbye";
string language = "English";
// Create an empty full-text stoplist
FullTextStopList stoplist = new FullTextStopList(db, "sampleStoplist");
stoplist.Create();
Console.WriteLine("FullTextStoplist '{0}' created", stoplist.Name);
// Add a stopword to the full-text stoplist
stoplist.AddStopWord(stopword, language);
Console.WriteLine("Stopword '{0}' added to Stoplist '{1}'", stopword, stoplist.Name);
fti.StopListName = "sampleStoplist";
// Create the full-text index and associate the full-text stoplist with it
fti.Create();
Console.WriteLine("FullTextIndex on Table '{0}' created", tab.Name);
fti.Drop();
Console.WriteLine("FullTextIndex on Table '{0}' dropped", tab.Name);
ftcat.Drop();
Console.WriteLine("FullTextCatalog '{0}' dropped", ftcat.Name);
tab.Drop();
Console.WriteLine("Table '{0}' created", tab.Name);
db.Drop();
Console.WriteLine("Database '{0}' dropped", db.Name);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
finally
{
Console.WriteLine();
}
}
public void Can_Use_FullTextIndex_Directly()
{
// First we create an instance of FullTextIndex
var index = new FullTextIndex();
// Then we add texts to the index
index.Add(_apodArticles);
// After adding we can Search for texts that match a Lucene query expression
// See https://lucene.apache.org/core/2_9_4/queryparsersyntax.html for a reference on Lucene query syntax
var result = index.Search("magellanic nebula visible in southern skies").ToList();
var expected = 10;
var actual = result.Count > 0 ? result[0] : -1;
Assert.AreEqual<int>(expected, actual); // Article #10 should come up on top
result = index.Search("swift-tuttle comet").ToList();
expected = 0;
actual = result.Count > 0 ? result[0] : -1;
Assert.AreEqual<int>(expected, actual); // Article #0 should come up on top
result = index.Search("china observation station in antartica").ToList();
expected = 14;
actual = result.Count > 0 ? result[0] : -1;
Assert.AreEqual<int>(expected, actual); // Article #14 should come up on top
}
public QueryTests()
{
this.index = new FullTextIndex <string>();
this.index.Index("A", "Some test text");
}
static void Main(string[] args)
{
using (var fti = new FullTextIndex(StorageEnvironmentOptions.ForPath("index"), new DefaultAnalyzer()))
{
var tasks = new List <Task>();
var queue = new BlockingCollection <string>(10 * 1000);
for (int i = 0; i < 1; i++)
{
tasks.Add(Task.Run(() =>
{
int indexerCount = 0;
using (var indexer = fti.CreateIndexer())
{
indexer.AutoFlush = false;
while (queue.IsCompleted == false)
{
string line;
try
{
line = queue.Take();
}
catch (InvalidOperationException)
{
break;
}
if (line == null)
{
break;
}
if (++indexerCount % (50 * 1000) == 0)
{
indexer.Flush();
}
indexer.NewIndexEntry();
indexer.AddField("Title", line);
}
indexer.Flush();
}
}));
}
var sp = Stopwatch.StartNew();
int count = 0;
using (var reader = new StreamReader(new GZipStream(File.OpenRead("titles.gz"), CompressionMode.Decompress)))
{
while (true)
{
var line = reader.ReadLine();
if (line == null)
{
break;
}
queue.Add(line);
if (++count % 50000 == 0)
{
Console.WriteLine("{0,10:#,#}: {1}", count, line);
}
}
queue.CompleteAdding();
Console.WriteLine("Reading {0:#,#} in {1}", count, sp.Elapsed);
}
while (Task.WaitAll(tasks.ToArray(), 1000) == false)
{
Console.Write("\r{0,10:#,#;;0} ", queue.Count);
}
Console.WriteLine("\rTotal " + sp.Elapsed);
}
}
public async Task InitializeAsync()
{
this.index = await CreateIndexAsync();
}
public void CanQueryWithInQuery()
{
using (var fullTextIndex = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("Name", "David Boike");
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren Eini");
indexer.NewIndexEntry();
indexer.AddField("Name", "Arava Eini");
indexer.NewIndexEntry();
indexer.AddField("Name", "Sean Epping");
indexer.NewIndexEntry();
indexer.AddField("Name", "Joe DeCock");
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
var results = searcher.QueryTop(new InQuery("Name", "joe", "epping", "boike"), 5);
Assert.Equal(3, results.Results.Length);
}
}
}
/// <summary>
/// This method will create a full-text index on a table with an associated stoplist
/// </summary>
public static void CreateFullTextIndex()
{
Console.WriteLine("Running CreateFullTextIndex Sample...");
try
{
Server server = new Server(".");
server.ConnectionContext.Connect();
Console.WriteLine("Connected to '{0}' server", server.Name);
Database db = new Database(server, "iFTSSampleDB2");
db.Create();
Console.WriteLine("Database '{0}' created", db.Name);
// Create a default full-text catalog for the database
FullTextCatalog ftcat = new FullTextCatalog(db, "ftcatalog");
ftcat.IsDefault = true;
ftcat.Create();
Console.WriteLine("FullTextCatalog '{0}' created", ftcat.Name);
// Add 2 columns to the table
Table tab = new Table(db, "tab");
tab.Columns.Add(new Column(tab, "col_fti", DataType.NVarChar(1000)));
Column col = new Column(tab, "col_unique", DataType.Int);
col.Nullable = false;
tab.Columns.Add(col);
// Create a table with a unique index on it
Index idxUnique = new Index(tab, "tab_unique_idx");
idxUnique.IndexKeyType = IndexKeyType.DriUniqueKey;
idxUnique.IndexedColumns.Add(new IndexedColumn(idxUnique, col.Name));
tab.Indexes.Add(idxUnique);
tab.Create();
Console.WriteLine("Table '{0}' created", tab.Name);
// Add a column to the full-text index and associate it with a unique index
FullTextIndex fti = new FullTextIndex(tab);
fti.IndexedColumns.Add(new FullTextIndexColumn(fti, "col_fti"));
fti.UniqueIndexName = idxUnique.Name;
string stopword = "goodbye";
string language = "English";
// Create an empty full-text stoplist
FullTextStopList stoplist = new FullTextStopList(db, "sampleStoplist");
stoplist.Create();
Console.WriteLine("FullTextStoplist '{0}' created", stoplist.Name);
// Add a stopword to the full-text stoplist
stoplist.AddStopWord(stopword, language);
Console.WriteLine("Stopword '{0}' added to Stoplist '{1}'", stopword, stoplist.Name);
fti.StopListName = "sampleStoplist";
// Create the full-text index and associate the full-text stoplist with it
fti.Create();
Console.WriteLine("FullTextIndex on Table '{0}' created", tab.Name);
fti.Drop();
Console.WriteLine("FullTextIndex on Table '{0}' dropped", tab.Name);
ftcat.Drop();
Console.WriteLine("FullTextCatalog '{0}' dropped", ftcat.Name);
tab.Drop();
Console.WriteLine("Table '{0}' created", tab.Name);
db.Drop();
Console.WriteLine("Database '{0}' dropped", db.Name);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
finally
{
Console.WriteLine();
}
}
public void CanIndexSingleValue()
{
using (var fti = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fti.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren Eini");
indexer.Flush();
}
}
}
public void CanQueryAndSort()
{
using (var fullTextIndex = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren Eini");
indexer.AddField("Email", "*****@*****.**");
indexer.NewIndexEntry();
indexer.AddField("Name", "Arava Eini");
indexer.AddField("Email", "*****@*****.**");
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
var results = searcher.QueryTop(new TermQuery("Name", "eini"), 5, sortBy: new Sorter("Email"));
Assert.Equal(2, results.Results[0].DocumentId);
Assert.Equal(1, results.Results[1].DocumentId);
}
}
}
public void CanQueryUsingSingleTerm()
{
using (var fullTextIndex = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren Eini");
indexer.AddField("Email", "*****@*****.**");
indexer.NewIndexEntry();
indexer.AddField("Name", "Arava Eini");
indexer.AddField("Email", "*****@*****.**");
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
Assert.Equal(1, searcher.Query(new TermQuery("Name", "oren")).Count());
Assert.Equal(0, searcher.Query(new TermQuery("Name", "rahien")).Count());
}
}
}
public void CanQueryOnEmptyindex()
{
using (var fti = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var searcher = fti.CreateSearcher())
{
Assert.Empty(searcher.Query(new TermQuery("Foo", "Arava")));
}
}
}
public void CanQueryUsingMissingTerm()
{
using (var fti = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fti.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren Eini");
indexer.Flush();
}
using (var searcher = fti.CreateSearcher())
{
Assert.Empty(searcher.Query(new TermQuery("Name", "Arava")));
}
}
}
public void CanQueryAndSortByTwoFields()
{
using (var fullTextIndex = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("QueryFor", "yes");
indexer.AddField("FirstName", "David");
indexer.AddField("LastName", "Boike");
indexer.NewIndexEntry();
indexer.AddField("QueryFor", "yes");
indexer.AddField("FirstName", "Natalie");
indexer.AddField("LastName", "Boike");
indexer.NewIndexEntry();
indexer.AddField("QueryFor", "NO");
indexer.AddField("FirstName", "NO");
indexer.AddField("LastName", "NO");
indexer.NewIndexEntry();
indexer.AddField("QueryFor", "yes");
indexer.AddField("FirstName", "Oren");
indexer.AddField("LastName", "Eini");
indexer.NewIndexEntry();
indexer.AddField("QueryFor", "yes");
indexer.AddField("FirstName", "Arava");
indexer.AddField("LastName", "Eini");
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
var results = searcher.QueryTop(new TermQuery("QueryFor", "yes"), 5, sortBy: new Sorter(new SortByTerm("LastName"), new SortByTerm("FirstName")));
Assert.Equal(4, results.Results.Length);
Console.WriteLine("{0}, {1}, {2}, {3}", results.Results[0].DocumentId, results.Results[1].DocumentId,
results.Results[2].DocumentId, results.Results[3].DocumentId);
Assert.Equal(1, results.Results[0].DocumentId);
Assert.Equal(2, results.Results[1].DocumentId);
Assert.Equal(5, results.Results[2].DocumentId);
Assert.Equal(4, results.Results[3].DocumentId);
}
}
}
public void CanDoPhraseQuery()
{
using (var fullTextIndex = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("PhraseText", "Not a match.", options: FieldOptions.TermPositions);
indexer.NewIndexEntry();
indexer.AddField("PhraseText", "RavenDB is a very cool database to work with.", options: FieldOptions.TermPositions);
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
var results = searcher.QueryTop(new PhraseQuery("PhraseText", "ravendb", "cool", "database")
{
Slop = 4
}, 5);
Assert.Equal(1, results.Results.Length);
Assert.Equal(2, results.Results[0].DocumentId);
}
using (var searcher = fullTextIndex.CreateSearcher())
{
var results = searcher.QueryTop(new PhraseQuery("PhraseText", "database", "cool", "ravendb")
{
Slop = 4
}, 5);
Assert.Equal(0, results.Results.Length);
}
}
}
public void CanIndexEmptyDocument()
{
using (var fti = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fti.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.Flush();
}
}
}
internal protected Root(AssocDB db)
: base(db.storage)
{
this.db = db;
attributes = db.storage.CreateIndex(typeof(string), true);
relations = db.storage.CreateIndex(typeof(long), false);
fullTextIndex = db.CreateFullTextIndex();
}
public void CanQueryUsingBooleanQuery()
{
using (var fullTextIndex = new FullTextIndex(StorageEnvironmentOptions.CreateMemoryOnly(), new DefaultAnalyzer()))
{
using (var indexer = fullTextIndex.CreateIndexer())
{
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren");
indexer.AddField("Email", "*****@*****.**");
indexer.NewIndexEntry();
indexer.AddField("Name", "Oren");
indexer.AddField("Email", "*****@*****.**");
indexer.Flush();
}
using (var searcher = fullTextIndex.CreateSearcher())
{
Assert.Equal(1, searcher.Query(new BooleanQuery(QueryOperator.And,
new TermQuery("Name", "oren"),
new TermQuery("Email", "*****@*****.**"))).Count());
Assert.Equal(2, searcher.Query(new BooleanQuery(QueryOperator.Or,
new TermQuery("Name", "oren"),
new TermQuery("Email", "*****@*****.**"))).Count());
}
}
}
