protected async Task PopulateIndexOneByOne(IFullTextIndex <string> index)
{
foreach (var page in WikipediaData.SampleData)
{
await index.AddAsync(page);
}
}
/// <summary>
/// Tests if any of the given substrings appear at the end of the string builder.
/// </summary>
/// <param name="builder">The builder to check within.</param>
/// <param name="replacementSetLookup">The potential replacements, keyed by the last character of the search text.</param>
/// <returns>
/// The text to replace at the end of the string builder. The match word of the word replacement will be null if no matches were found.
/// </returns>
public static WordReplacement EndsWith(this StringBuilder builder, IFullTextIndex <WordReplacement> replacementSetLookup)
{
var length = builder.Length;
if (length > 3)
{
using (var navigator = replacementSetLookup.Snapshot.CreateNavigator())
{
if (navigator.Process(builder[builder.Length - 1]))
{
var bestMatch = IntermediateQueryResult.Empty;
for (var i = builder.Length - 2; i >= 0; i--)
{
if (!navigator.Process(builder[i]))
{
break;
}
if (navigator.HasExactMatches)
{
bestMatch = navigator.GetExactMatches();
}
}
if (bestMatch.Matches.Count > 0)
{
return(replacementSetLookup.Items.GetMetadata(bestMatch.Matches[0].ItemId).Item);
}
}
}
}
return(default);
public static void Visit(this IFullTextIndex index, IIndexVisitor visitor)
{
foreach (var term in index.GetTerms(new DfaTermMatcher(new AnyMatcher <char>())))
{
if (!visitor.VisitTerm(term))
{
break;
}
}
}
public static void Visit(this IFullTextIndex index, IIndexVisitor visitor)
{
foreach (var term in index.GetTerms("*"))
{
if (!visitor.VisitTerm(term))
{
break;
}
}
}
private void TextIndex(IFullTextIndex index)
{
Assert.Equal(new TextPosition[]
{
TextPosition.P(0, 5),
TextPosition.P(6, 5)
},
index.GetPositions(1, 1));
Assert.Equal(new TextPosition[]
{
TextPosition.P(0, 5),
TextPosition.P(6, 9),
TextPosition.P(16, 6)
},
index.GetPositions(2, 1));
Assert.Equal(new TextPosition[]
{
TextPosition.P(0, 11),
TextPosition.P(12, 2),
TextPosition.P(15, 4),
TextPosition.P(20, 8),
TextPosition.P(30, 8),
TextPosition.P(39, 3),
TextPosition.P(43, 4),
TextPosition.P(49, 4),
TextPosition.P(54, 2),
TextPosition.P(57, 9)
},
index.GetPositions(6, 1));
Assert.Equal("Programming is very exciting. Programs can help. This is fantastic!!!", index.GetText(6, 1).ReadToEnd());
var doc = new TextDocument(index.GetText(6, 1).ReadToEnd(),
index.GetPositions(6, 1));
Assert.Equal("Programming", doc.Token(0));
Assert.Equal("is", doc.Token(1));
Assert.Equal("very", doc.Token(2));
Assert.Equal("exciting", doc.Token(3));
Assert.Equal("Programs", doc.Token(4));
Assert.Equal("can", doc.Token(5));
Assert.Equal("help", doc.Token(6));
Assert.Equal("This", doc.Token(7));
Assert.Equal("is", doc.Token(8));
Assert.Equal("fantastic", doc.Token(9));
Assert.Equal("Programming", doc.TokenAt(3));
Assert.Equal("is", doc.TokenAt(13));
Assert.Equal("very", doc.TokenAt(15));
Assert.Equal("fantastic", doc.TokenAt(60));
}
public Reindexer(
IFullTextIndex fullTextIndex,
IQueryParser queryParser,
ITokenizer tokenizer,
IFileWatcherService fileWatcherService,
IDisposable disposable
)
{
this.fullTextIndex = fullTextIndex;
this.queryParser = queryParser;
this.tokenizer = tokenizer;
this.fileWatcherService = fileWatcherService;
this.disposable = disposable;
indexChangesSubscription = fileWatcherService.Changes
.ObserveOn(ThreadPoolScheduler.Instance)
.Subscribe(IndexChanges);
}
public IEnumerable <long> Execute(IFullTextIndex index)
{
var queryEvaluations = new Queue <IEnumerable <long> >();
foreach (var query in Queries)
{
var queryEvaluation = query.Execute(index);
queryEvaluations.Enqueue(queryEvaluation);
}
while (queryEvaluations.Count > 1)
{
var first = queryEvaluations.Dequeue();
var second = queryEvaluations.Dequeue();
queryEvaluations.Enqueue(first.Intersect(second));
}
return(queryEvaluations.Dequeue());
}
public IEnumerable <SearchResult <TKey> > Execute <TKey>(IFullTextIndex <TKey> index)
{
if (index is null)
{
throw new ArgumentNullException(nameof(index));
}
if (this.Root == null)
{
yield break;
}
var matches = this.Root.Evaluate(() => new IndexNavigator(index.Root)).Matches;
var results = new Dictionary <int, List <FieldMatch> >();
foreach (var match in matches)
{
if (!results.TryGetValue(match.ItemId, out var itemResults))
{
itemResults = new List <FieldMatch>();
results[match.ItemId] = itemResults;
}
itemResults.AddRange(match.FieldMatches);
}
foreach (var itemResults in matches)
{
var item = index.IdPool.GetItemForId(itemResults.ItemId);
yield return(new SearchResult <TKey>(
item,
itemResults.FieldMatches.Select(m => new FieldSearchResult(
index.FieldLookup.GetFieldForId(m.FieldId),
m.GetWordLocations()))
.ToList()));
}
}
protected async Task PopulateIndexAsync(IFullTextIndex <string> index)
{
await index.AddRangeAsync(WikipediaData.SampleData);
}
public IEnumerable <long> Execute(IFullTextIndex index)
{
return(Array.Empty <long>());
}
public void SetUp()
{
fullTextIndex = new FullTextIndexFactory().Create();
}
private void TestTermSearch(IFullTextIndex index)
{
var postings = index.GetPostingList("this").ToString();
Assert.Equal("[3,1,1], [4,1,1], [5,1,1], [6,1,8]", postings);
}
public static IEnumerable <IPostingList> GetPostingLists(this IFullTextIndex index, string word, int distance)
{
return(index.GetTerms(word, distance).Select(p => index.PostingLists.Get(p.Value)));
}
public static IEnumerable <DictionaryTerm> GetTerms(this IFullTextIndex index, string word, int distance)
{
var matcher = new DfaTermMatcher(new LevenshteinMatcher(word, distance));
return(index.GetTerms(matcher));
}
public PrintVisitor(IFullTextIndex index)
{
this.index = index;
this.Terms = 0;
}
public async Task InitializeAsync()
{
this.index = new FullTextIndexBuilder <string>().Build();
await this.index.AddAsync("A", "Some test text");
}
public FullTextQueryCompiler(IFullTextIndex index)
{
this.index = index;
this.maxTokenLength = index.Header.MaxTokenSize;
}
public static IPostingList GetPostingList(IFullTextIndex index, string term)
{
var dictionaryTerm = index.Dictionary.GetTerms(new DfaTermMatcher(new WildcardMatcher(term, index.Header.MaxTokenSize))).Single();
return(index.PostingLists.Get(dictionaryTerm.Value));
}
public static IEnumerable <DictionaryTerm> GetTerms(this IFullTextIndex index, string wildcardPattern)
{
var matcher = new DfaTermMatcher(new WildcardMatcher(wildcardPattern, index.Header.MaxTokenSize));
return(index.GetTerms(matcher));
}
public IEnumerable <long> Execute(IFullTextIndex index)
{
return(index.IndexedDocuments.Except(Query.Execute(index)));
}
public static IEnumerable <IPostingList> GetPostingLists(this IFullTextIndex index, string wildcardPattern)
{
return(index.GetTerms(wildcardPattern).Select(p => index.PostingLists.Get(p.Value)));
}
public async Task SetUp()
{
this.index = CreateNewIndex(4);
await this.PopulateIndexAsync(this.index);
}
public static IPostingList GetPostingList(this IFullTextIndex index, string term)
{
return(GetPostingLists(index, term).Single());
}
public PrintVisitor(IFullTextIndex index)
{
this.index = index;
}
public IEnumerable <long> Execute(IFullTextIndex index)
{
return(index.GetMatchingDocuments(Term));
}
private void TestTermSearch(IFullTextIndex index, string query, string expected)
{
var postings = index.Compile(query).ExecuteToString();
Assert.Equal(expected, postings);
}
