internal IndexedFieldLookup(
IEnumerable <IFieldTokenizationOptions> fieldTokenizationOptions,
ITokenizerFactory tokenizerFactory,
TokenizationOptions defaultTokenizationOptions)
{
if (fieldTokenizationOptions is null)
{
throw new ArgumentNullException(nameof(fieldTokenizationOptions));
}
if (tokenizerFactory is null)
{
throw new ArgumentNullException(nameof(tokenizerFactory));
}
if (defaultTokenizationOptions is null)
{
throw new ArgumentNullException(nameof(defaultTokenizationOptions));
}
foreach (var field in fieldTokenizationOptions)
{
this.RegisterField(field, tokenizerFactory, defaultTokenizationOptions);
}
}
/// <summary>
/// Specifies the default tokenization options that should be used when searching or indexing
/// when no other options are provided.
/// </summary>
public FullTextIndexBuilder <TKey> WithDefaultTokenizationOptions(Func <TokenizationOptionsBuilder, TokenizationOptionsBuilder> optionsBuilder)
{
if (optionsBuilder is null)
{
throw new ArgumentNullException(nameof(optionsBuilder));
}
this.defaultTokenizationOptions = optionsBuilder.BuildOptionsOrDefault();
return(this);
}
/// <summary>
/// Builds an <see cref="ITokenizer"/> instance matching the current configuration.
/// </summary>
public ITokenizer Build()
{
var options = new TokenizationOptions()
{
SplitOnPunctuation = this.splitOnPunctuation,
AccentInsensitive = this.accentInsensitive,
CaseInsensitive = this.caseInsensitive,
Stemming = this.stemming
};
if (this.additionalSplitCharacters != null)
{
options.AdditionalSplitCharacters = this.additionalSplitCharacters;
}
return(this.factory(options));
}
internal FullTextIndex(
IndexOptions indexOptions,
ConfiguredItemTokenizationOptions <TKey> itemTokenizationOptions,
IIndexNodeFactory indexNodeFactory,
ITokenizerFactory tokenizerFactory,
IQueryParser queryParser,
TokenizationOptions defaultTokenizationOptions,
Func <IIndexSnapshot <TKey>, Task>[]?indexModifiedActions)
{
this.indexOptions = indexOptions;
this.itemTokenizationOptions = itemTokenizationOptions ?? throw new ArgumentNullException(nameof(itemTokenizationOptions));
this.IndexNodeFactory = indexNodeFactory ?? throw new ArgumentNullException(nameof(indexNodeFactory));
this.tokenizerFactory = tokenizerFactory ?? throw new ArgumentNullException(nameof(tokenizerFactory));
this.queryParser = queryParser ?? throw new ArgumentNullException(nameof(queryParser));
this.defaultTokenizationOptions = defaultTokenizationOptions ?? throw new ArgumentNullException(nameof(defaultTokenizationOptions));
this.indexModifiedActions = indexModifiedActions;
this.idPool = new IdPool <TKey>();
this.FieldLookup = new IndexedFieldLookup(
this.itemTokenizationOptions.GetAllConfiguredFields(),
tokenizerFactory,
defaultTokenizationOptions);
this.Root = this.IndexNodeFactory.CreateRootNode();
}
private void RegisterField(IFieldTokenizationOptions fieldOptions, ITokenizerFactory tokenizerFactory, TokenizationOptions defaultTokenizationOptions)
{
var fieldName = fieldOptions.Name;
if (this.fieldToDetailsLookup.ContainsKey(fieldOptions.Name))
{
throw new LiftiException(ExceptionMessages.FieldNameAlreadyUsed, fieldName);
}
var newId = Interlocked.Increment(ref nextId);
if (newId > byte.MaxValue)
{
throw new LiftiException(ExceptionMessages.MaximumDistinctFieldsIndexReached);
}
var id = (byte)newId;
var fieldTokenizationOptions = fieldOptions.TokenizationOptions ?? defaultTokenizationOptions;
this.fieldToDetailsLookup[fieldName] = new IndexedFieldDetails((byte)id, tokenizerFactory.Create(fieldTokenizationOptions));
this.idToFieldLookup[id] = fieldName;
}