private static void AddGram(string text, global::Lucene.Net.Documents.Document doc, int ng1, int ng2)
{
var table = GramsTable;
int len = text.Length;
for (int ng = ng1; ng <= ng2; ng++)
{
string end = null;
for (int i = 0; i < len - ng + 1; i++)
{
string gram = text.Substring(i, (i + ng) - (i));
doc.Add(new Field(table[ng].Gram, gram, Field.Store.NO, Field.Index.NOT_ANALYZED));
if (i == 0)
{
doc.Add(new Field(table[ng].Start, gram, Field.Store.NO, Field.Index.NOT_ANALYZED));
}
end = gram;
}
if (end != null)
{
// may not be present if len==ng1
doc.Add(new Field(table[ng].End, end, Field.Store.NO, Field.Index.NOT_ANALYZED));
}
}
}
private static global::Lucene.Net.Documents.Document CreateDocument(string text, int ng1, int ng2)
{
global::Lucene.Net.Documents.Document doc = new global::Lucene.Net.Documents.Document();
doc.Add(new Field(FWord, text, Field.Store.YES, Field.Index.NOT_ANALYZED)); // orig term
AddGram(text, doc, ng1, ng2);
return(doc);
}
public override void ToObject(global::Lucene.Net.Documents.Document source, global::Lucene.Net.Linq.IQueryExecutionContext context, T target)
{
var id = source.GetField("__pieId").StringValue;
target.PieId(id);
base.ToObject(source, context, target);
}
/// <summary>
/// Recupera os dados do elemento com base no documento.
/// </summary>
/// <param name="doc"></param>
/// <returns></returns>
protected Element GetElementFromDocument(global::Lucene.Net.Documents.Document doc)
{
var uidField = doc.GetField("Uid");
var uid = int.Parse(uidField.StringValue());
return(_dataRepository[uid]);
}
public FilterResult Apply(global::Lucene.Net.Documents.Document document, string key, IState state)
{
var doc = _retriever.DirectGet(document, key, DocumentFields.All, state);
if (doc == null)
{
_skippedResults.Value++;
return(FilterResult.Skipped);
}
if (_scannedDocuments.Value >= _query.FilterLimit)
{
return(FilterResult.LimitReached);
}
_scannedDocuments.Value++;
object self = _filterScriptRun.Translate(_documentsContext, doc);
using (_queryTimings?.For(nameof(QueryTimingsScope.Names.Filter)))
using (var result = _filterScriptRun.Run(_documentsContext, _documentsContext, "execute", new[] { self, _query.QueryParameters }, _queryTimings))
{
if (result.BooleanValue == true)
{
return(FilterResult.Accepted);
}
_skippedResults.Value++;
return(FilterResult.Skipped);
}
}
public void AddDocument(global::Lucene.Net.Documents.Document doc, Analyzer analyzer, IState state)
{
var fieldables = doc.GetFieldables(_field);
if (fieldables == null)
{
return;
}
foreach (var fieldable in fieldables)
{
if (fieldable == null)
{
continue;
}
TextReader reader;
var str = fieldable.StringValue(state);
if (!string.IsNullOrEmpty(str))
{
reader = new StringReader(str);
}
else
{
// We are reusing the fieldable for indexing. Instead of recreating it, we just reset the underlying text reader.
reader = fieldable.ReaderValue;
if (reader is ReusableStringReader stringReader)
{
if (stringReader.Length == 0)
{
continue;
}
stringReader.Reset();
}
else if (reader is StreamReader streamReader)
{
if (streamReader.BaseStream.Length == 0)
{
continue;
}
streamReader.BaseStream.Position = 0;
}
else
{
continue;
}
}
var tokenStream = analyzer.ReusableTokenStream(_field, reader);
while (tokenStream.IncrementToken())
{
var word = tokenStream.GetAttribute <ITermAttribute>().Term;
// Index
int len = word.Length;
if (len < 3)
{
continue; // too short we bail but "too long" is fine...
}
// Early skip avoiding allocation of terms and searching.
if (_alreadySeen.Contains(word))
{
continue;
}
_indexSearcher ??= new IndexSearcher(_directory, true, state);
if (_indexSearcher.DocFreq(_fWordTerm.CreateTerm(word), state) <= 0)
{
// the word does not exist in the gramindex
int min = GetMin(len);
_indexWriter.AddDocument(CreateDocument(word, min, min + 1), state);
}
_alreadySeen.Add(word);
}
}
}
public void AddDocument(global::Lucene.Net.Documents.Document doc, Analyzer a, IState state)
{
_indexWriter.AddDocument(doc, a, state);
}
private Dictionary <string, Dictionary <string, string[]> > GetHighlighterResults(IndexQueryServerSide query, IndexSearcher searcher, ScoreDoc scoreDoc, Document document, global::Lucene.Net.Documents.Document luceneDocument, JsonOperationContext context)
{
Debug.Assert(_highlighter != null);
Debug.Assert(_highlighterQuery != null);
var results = new Dictionary <string, Dictionary <string, string[]> >();
foreach (var highlighting in query.Metadata.Highlightings)
{
var fieldName = highlighting.Field.Value;
var indexFieldName = query.Metadata.IsDynamic
? AutoIndexField.GetSearchAutoIndexFieldName(fieldName)
: fieldName;
var fragments = _highlighter.GetBestFragments(
_highlighterQuery,
searcher.IndexReader,
scoreDoc.Doc,
indexFieldName,
highlighting.FragmentLength,
highlighting.FragmentCount,
_state);
if (fragments == null || fragments.Length == 0)
{
continue;
}
var options = highlighting.GetOptions(context, query.QueryParameters);
string key;
if (options != null && string.IsNullOrWhiteSpace(options.GroupKey) == false)
{
key = luceneDocument.Get(options.GroupKey, _state);
}
else
{
key = document.Id;
}
if (results.TryGetValue(fieldName, out var result) == false)
{
results[fieldName] = result = new Dictionary <string, string[]>();
}
if (result.TryGetValue(key, out var innerResult))
{
Array.Resize(ref innerResult, innerResult.Length + fragments.Length);
Array.Copy(fragments, 0, innerResult, innerResult.Length, fragments.Length);
}
else
{
result[key] = fragments;
}
}
return(results);
}
private ExplanationResult GetQueryExplanations(ExplanationOptions options, Query luceneQuery, IndexSearcher searcher, ScoreDoc scoreDoc, Document document, global::Lucene.Net.Documents.Document luceneDocument)
{
string key;
var hasGroupKey = options != null && string.IsNullOrWhiteSpace(options.GroupKey) == false;
if (_indexType.IsMapReduce())
{
if (hasGroupKey)
{
key = luceneDocument.Get(options.GroupKey, _state);
if (key == null && document.Data.TryGet(options.GroupKey, out object value))
{
key = value?.ToString();
}
}
else
{
key = luceneDocument.Get(Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName, _state);
}
}
else
{
key = hasGroupKey
? luceneDocument.Get(options.GroupKey, _state)
: document.Id;
}
return(new ExplanationResult
{
Key = key,
Explanation = searcher.Explain(luceneQuery, scoreDoc.Doc, _state)
});
}
public override void ToDocument(T source, global::Lucene.Net.Documents.Document target)
{
base.ToDocument(source, target);
target.Add(new Field("__pieId", source.PieId(), Field.Store.YES, Field.Index.NOT_ANALYZED));
}
public void AddDocument(global::Lucene.Net.Documents.Document doc, Analyzer a)
{
indexWriter.AddDocument(doc, a);
}
public void AddDocument(global::Lucene.Net.Documents.Document doc)
{
indexWriter.AddDocument(doc);
}
public override void ToObject(global::Lucene.Net.Documents.Document source, global::Lucene.Net.Linq.IQueryExecutionContext context, T target)
{
base.ToObject(source, context, target);
}
private ExplanationResult GetQueryExplanations(ExplanationOptions options, Query luceneQuery, IndexSearcher searcher, ScoreDoc scoreDoc, Document document, global::Lucene.Net.Documents.Document luceneDocument)
{
string key;
if (options != null && string.IsNullOrWhiteSpace(options.GroupKey) == false)
{
key = luceneDocument.Get(options.GroupKey, _state);
}
else
{
key = document.Id;
}
return(new ExplanationResult
{
Key = key,
Explanation = searcher.Explain(luceneQuery, scoreDoc.Doc, _state)
});
}
