public IndexReadOperation(Index index, LuceneVoronDirectory directory, IndexSearcherHolder searcherHolder, QueryBuilderFactories queryBuilderFactories, Transaction readTransaction)
: base(index, LoggingSource.Instance.GetLogger <IndexReadOperation>(index._indexStorage.DocumentDatabase.Name))
{
try
{
_analyzer = CreateAnalyzer(() => new LowerCaseKeywordAnalyzer(), index.Definition.IndexFields, forQuerying: true);
}
catch (Exception e)
{
throw new IndexAnalyzerException(e);
}
_queryBuilderFactories = queryBuilderFactories;
_maxNumberOfOutputsPerDocument = index.MaxNumberOfOutputsPerDocument;
_indexType = index.Type;
_indexHasBoostedFields = index.HasBoostedFields;
_releaseReadTransaction = directory.SetTransaction(readTransaction, out _state);
_releaseSearcher = searcherHolder.GetSearcher(readTransaction, _state, out _searcher);
}
public IndexFacetedReadOperation(string indexName,
Dictionary <string, IndexField> fields,
LuceneVoronDirectory directory,
IndexSearcherHolder searcherHolder,
Transaction readTransaction,
DocumentDatabase documentDatabase)
: base(indexName, LoggingSource.Instance.GetLogger <IndexFacetedReadOperation>(documentDatabase.Name))
{
try
{
_analyzer = CreateAnalyzer(() => new LowerCaseKeywordAnalyzer(), fields, forQuerying: true);
}
catch (Exception e)
{
throw new IndexAnalyzerException(e);
}
_releaseReadTransaction = directory.SetTransaction(readTransaction, out _state);
_currentStateHolder = searcherHolder.GetStateHolder(readTransaction);
_searcher = _currentStateHolder.GetIndexSearcher(_state);
}
public IndexWriteOperation(Index index, LuceneVoronDirectory directory, LuceneDocumentConverterBase converter, Transaction writeTransaction, LuceneIndexPersistence persistence)
: base(index, LoggingSource.Instance.GetLogger <IndexWriteOperation>(index._indexStorage.DocumentDatabase.Name))
{
_converter = converter;
DocumentDatabase = index._indexStorage.DocumentDatabase;
try
{
_analyzer = CreateAnalyzer(index, index.Definition);
}
catch (Exception e)
{
throw new IndexAnalyzerException(e);
}
try
{
_releaseWriteTransaction = directory.SetTransaction(writeTransaction, out _state);
_writer = persistence.EnsureIndexWriter(_state);
_suggestionsWriters = persistence.EnsureSuggestionIndexWriter(_state);
_hasSuggestions = _suggestionsWriters.Count > 0;
_locker = directory.MakeLock("writing-to-index.lock");
_directory = directory;
if (_locker.Obtain() == false)
{
throw new InvalidOperationException($"Could not obtain the 'writing-to-index' lock for '{_indexName}' index.");
}
}
catch (Exception e) when(e.IsOutOfMemory())
{
throw;
}
catch (Exception e)
{
throw new IndexWriteException(e);
}
}
public IndexFacetedReadOperation(Index index,
IndexDefinitionBaseServerSide indexDefinition,
LuceneVoronDirectory directory,
IndexSearcherHolder searcherHolder,
QueryBuilderFactories queryBuilderFactories,
Transaction readTransaction,
DocumentDatabase documentDatabase)
: base(index, LoggingSource.Instance.GetLogger <IndexFacetedReadOperation>(documentDatabase.Name))
{
try
{
_analyzer = CreateAnalyzer(index, indexDefinition, forQuerying: true);
}
catch (Exception e)
{
throw new IndexAnalyzerException(e);
}
_queryBuilderFactories = queryBuilderFactories;
_releaseReadTransaction = directory.SetTransaction(readTransaction, out _state);
_releaseSearcher = searcherHolder.GetSearcher(readTransaction, _state, out _searcher);
}
public MoreLikeThisQueryResult ExecuteMoreLikeThisQuery(MoreLikeThisQuery query, TransactionInformation transactionInformation, int pageSize = 25)
{
if (query == null)
{
throw new ArgumentNullException("query");
}
var index = database.IndexStorage.GetIndexInstance(query.IndexName);
if (index == null)
{
throw new InvalidOperationException("The index " + query.IndexName + " cannot be found");
}
if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0)
{
throw new InvalidOperationException("The document id or map group fields are mandatory");
}
IndexSearcher searcher;
using (database.IndexStorage.GetCurrentIndexSearcher(index.indexId, out searcher))
{
var documentQuery = new BooleanQuery();
if (string.IsNullOrEmpty(query.DocumentId) == false)
{
documentQuery.Add(new TermQuery(new Term(Constants.DocumentIdFieldName, query.DocumentId.ToLowerInvariant())), Occur.MUST);
}
foreach (string key in query.MapGroupFields.Keys)
{
documentQuery.Add(new TermQuery(new Term(key, query.MapGroupFields[key])), Occur.MUST);
}
var td = searcher.Search(documentQuery, 1);
// get the current Lucene docid for the given RavenDB doc ID
if (td.ScoreDocs.Length == 0)
{
throw new InvalidOperationException("Document " + query.DocumentId + " could not be found");
}
var ir = searcher.IndexReader;
var mlt = new RavenMoreLikeThis(ir);
AssignParameters(mlt, query);
if (string.IsNullOrWhiteSpace(query.StopWordsDocumentId) == false)
{
var stopWordsDoc = database.Documents.Get(query.StopWordsDocumentId, null);
if (stopWordsDoc == null)
{
throw new InvalidOperationException("Stop words document " + query.StopWordsDocumentId + " could not be found");
}
var stopWordsSetup = stopWordsDoc.DataAsJson.JsonDeserialization <StopWordsSetup>();
if (stopWordsSetup.StopWords != null)
{
var stopWords = stopWordsSetup.StopWords;
var ht = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase);
foreach (var stopWord in stopWords)
{
ht.Add(stopWord);
}
mlt.SetStopWords(ht);
}
}
var fieldNames = query.Fields ?? GetFieldNames(ir);
mlt.SetFieldNames(fieldNames);
var toDispose = new List <Action>();
RavenPerFieldAnalyzerWrapper perFieldAnalyzerWrapper = null;
try
{
perFieldAnalyzerWrapper = index.CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose, true);
mlt.Analyzer = perFieldAnalyzerWrapper;
var mltQuery = mlt.Like(td.ScoreDocs[0].Doc);
var tsdc = TopScoreDocCollector.Create(pageSize, true);
if (string.IsNullOrWhiteSpace(query.AdditionalQuery) == false)
{
var additionalQuery = QueryBuilder.BuildQuery(query.AdditionalQuery, perFieldAnalyzerWrapper);
mltQuery = new BooleanQuery
{
{ mltQuery, Occur.MUST },
{ additionalQuery, Occur.MUST },
};
}
searcher.Search(mltQuery, tsdc);
var hits = tsdc.TopDocs().ScoreDocs;
var jsonDocuments = GetJsonDocuments(query, searcher, index, query.IndexName, hits, td.ScoreDocs[0].Doc);
var result = new MultiLoadResult();
var includedEtags = new List <byte>(jsonDocuments.SelectMany(x => x.Etag.ToByteArray()));
includedEtags.AddRange(database.Indexes.GetIndexEtag(query.IndexName, null).ToByteArray());
var loadedIds = new HashSet <string>(jsonDocuments.Select(x => x.Key));
var addIncludesCommand = new AddIncludesCommand(database, transactionInformation, (etag, includedDoc) =>
{
includedEtags.AddRange(etag.ToByteArray());
result.Includes.Add(includedDoc);
}, query.Includes ?? new string[0], loadedIds);
idsToLoad = new HashSet <string>();
database.TransactionalStorage.Batch(actions =>
{
documentRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, query.TransformerParameters, idsToLoad);
using (new CurrentTransformationScope(database, documentRetriever))
{
foreach (var document in ProcessResults(query, jsonDocuments, database.WorkContext.CancellationToken))
{
result.Results.Add(document);
addIncludesCommand.Execute(document);
}
}
});
addIncludesCommand.AlsoInclude(idsToLoad);
var computeHash = Encryptor.Current.Hash.Compute16(includedEtags.ToArray());
Etag computedEtag = Etag.Parse(computeHash);
return(new MoreLikeThisQueryResult
{
Etag = computedEtag,
Result = result,
});
}
finally
{
if (perFieldAnalyzerWrapper != null)
{
perFieldAnalyzerWrapper.Close();
}
foreach (var action in toDispose)
{
action();
}
}
}
}
protected static RavenPerFieldAnalyzerWrapper CreateAnalyzer(Func <Analyzer> createDefaultAnalyzer, IndexDefinitionBase indexDefinition, bool forQuerying = false)
{
if (indexDefinition.IndexFields.ContainsKey(Constants.Documents.Indexing.Fields.AllFields))
{
throw new InvalidOperationException($"Detected '{Constants.Documents.Indexing.Fields.AllFields}'. This field should not be present here, because inheritance is done elsewhere.");
}
var hasDefaultFieldOptions = false;
Analyzer defaultAnalyzerToUse = null;
RavenStandardAnalyzer standardAnalyzer = null;
KeywordAnalyzer keywordAnalyzer = null;
Analyzer defaultAnalyzer = null;
if (indexDefinition is MapIndexDefinition mid)
{
if (mid.IndexDefinition.Fields.TryGetValue(Constants.Documents.Indexing.Fields.AllFields, out var value))
{
hasDefaultFieldOptions = true;
switch (value.Indexing)
{
case FieldIndexing.Exact:
defaultAnalyzerToUse = keywordAnalyzer = new KeywordAnalyzer();
break;
case FieldIndexing.Search:
if (value.Analyzer != null)
{
defaultAnalyzerToUse = GetAnalyzer(Constants.Documents.Indexing.Fields.AllFields, value.Analyzer, forQuerying);
}
if (defaultAnalyzerToUse == null)
{
defaultAnalyzerToUse = standardAnalyzer = new RavenStandardAnalyzer(Version.LUCENE_29);
}
break;
default:
// explicitly ignore all other values
break;
}
}
}
if (defaultAnalyzerToUse == null)
{
defaultAnalyzerToUse = defaultAnalyzer = createDefaultAnalyzer();
}
var perFieldAnalyzerWrapper = new RavenPerFieldAnalyzerWrapper(defaultAnalyzerToUse);
foreach (var field in indexDefinition.IndexFields)
{
var fieldName = field.Value.Name;
switch (field.Value.Indexing)
{
case FieldIndexing.Exact:
if (keywordAnalyzer == null)
{
keywordAnalyzer = new KeywordAnalyzer();
}
perFieldAnalyzerWrapper.AddAnalyzer(fieldName, keywordAnalyzer);
break;
case FieldIndexing.Search:
var analyzer = GetAnalyzer(fieldName, field.Value.Analyzer, forQuerying);
if (analyzer != null)
{
perFieldAnalyzerWrapper.AddAnalyzer(fieldName, analyzer);
continue;
}
AddStandardAnalyzer(fieldName);
break;
case FieldIndexing.Default:
if (hasDefaultFieldOptions)
{
// if we have default field options then we need to take into account overrides for regular fields
if (defaultAnalyzer == null)
{
defaultAnalyzer = createDefaultAnalyzer();
}
perFieldAnalyzerWrapper.AddAnalyzer(fieldName, defaultAnalyzer);
continue;
}
break;
}
}
return(perFieldAnalyzerWrapper);
void AddStandardAnalyzer(string fieldName)
{
if (standardAnalyzer == null)
{
standardAnalyzer = new RavenStandardAnalyzer(Version.LUCENE_29);
}
perFieldAnalyzerWrapper.AddAnalyzer(fieldName, standardAnalyzer);
}
}
private static Query Parse(string q)
{
using (var defaultAnalyzer = new KeywordAnalyzer())
using (var perFieldAnalyzerWrapper = new RavenPerFieldAnalyzerWrapper(defaultAnalyzer))
return(QueryBuilder.BuildQuery(q, perFieldAnalyzerWrapper));
}
public SimpleFilesQueryParser(Analyzer analyzer)
: base(Version.LUCENE_29, string.Empty, analyzer)
{
fieldAnalyzer = new RavenPerFieldAnalyzerWrapper(fieldAnalyzer);
}
public MoreLikeThisQueryResult ExecuteMoreLikeThisQuery(MoreLikeThisQuery query, TransactionInformation transactionInformation, int pageSize = 25, string[] include = null)
{
if (query == null)
{
throw new ArgumentNullException("query");
}
var index = database.IndexStorage.GetIndexInstance(query.IndexName);
if (index == null)
{
throw new InvalidOperationException("The index " + query.IndexName + " cannot be found");
}
if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0)
{
throw new InvalidOperationException("The document id or map group fields are mandatory");
}
IndexSearcher searcher;
using (database.IndexStorage.GetCurrentIndexSearcher(query.IndexName, out searcher))
{
var documentQuery = new BooleanQuery();
if (string.IsNullOrEmpty(query.DocumentId) == false)
{
documentQuery.Add(new TermQuery(new Term(Constants.DocumentIdFieldName, query.DocumentId.ToLowerInvariant())), Occur.MUST);
}
foreach (string key in query.MapGroupFields.Keys)
{
documentQuery.Add(new TermQuery(new Term(key, query.MapGroupFields[key])), Occur.MUST);
}
var td = searcher.Search(documentQuery, 1);
// get the current Lucene docid for the given RavenDB doc ID
if (td.ScoreDocs.Length == 0)
{
throw new InvalidOperationException("Document " + query.DocumentId + " could not be found");
}
var ir = searcher.IndexReader;
var mlt = new RavenMoreLikeThis(ir);
AssignParameters(mlt, query);
if (string.IsNullOrWhiteSpace(query.StopWordsDocumentId) == false)
{
var stopWordsDoc = database.Get(query.StopWordsDocumentId, null);
if (stopWordsDoc == null)
{
throw new InvalidOperationException("Stop words document " + query.StopWordsDocumentId + " could not be found");
}
var stopWordsSetup = stopWordsDoc.DataAsJson.JsonDeserialization <StopWordsSetup>();
if (stopWordsSetup.StopWords != null)
{
var stopWords = stopWordsSetup.StopWords;
var ht = new Hashtable(StringComparer.InvariantCultureIgnoreCase);
foreach (var stopWord in stopWords)
{
ht[stopWord] = stopWord;
}
mlt.SetStopWords(ht);
}
}
var fieldNames = query.Fields ?? GetFieldNames(ir);
mlt.SetFieldNames(fieldNames);
var toDispose = new List <Action>();
RavenPerFieldAnalyzerWrapper perFieldAnalyzerWrapper = null;
try
{
perFieldAnalyzerWrapper = index.CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose, true);
mlt.SetAnalyzer(perFieldAnalyzerWrapper);
var mltQuery = mlt.Like(td.ScoreDocs[0].Doc);
var tsdc = TopScoreDocCollector.Create(pageSize, true);
searcher.Search(mltQuery, tsdc);
var hits = tsdc.TopDocs().ScoreDocs;
var jsonDocuments = GetJsonDocuments(query, searcher, index, query.IndexName, hits, td.ScoreDocs[0].Doc);
var result = new MultiLoadResult();
var includedEtags = new List <byte>(jsonDocuments.SelectMany(x => x.Etag.Value.ToByteArray()));
includedEtags.AddRange(database.GetIndexEtag(query.IndexName, null).ToByteArray());
var loadedIds = new HashSet <string>(jsonDocuments.Select(x => x.Key));
var addIncludesCommand = new AddIncludesCommand(database, transactionInformation, (etag, includedDoc) =>
{
includedEtags.AddRange(etag.ToByteArray());
result.Includes.Add(includedDoc);
}, include ?? new string[0], loadedIds);
foreach (var jsonDocument in jsonDocuments)
{
result.Results.Add(jsonDocument.ToJson());
addIncludesCommand.Execute(jsonDocument.DataAsJson);
}
Guid computedEtag;
using (var md5 = MD5.Create())
{
var computeHash = md5.ComputeHash(includedEtags.ToArray());
computedEtag = new Guid(computeHash);
}
return(new MoreLikeThisQueryResult
{
Etag = computedEtag,
Result = result,
});
}
finally
{
if (perFieldAnalyzerWrapper != null)
{
perFieldAnalyzerWrapper.Close();
}
foreach (var action in toDispose)
{
action();
}
}
}
}
