public IndexReadOperation(Index index, LuceneVoronDirectory directory, IndexSearcherHolder searcherHolder, QueryBuilderFactories queryBuilderFactories, Transaction readTransaction) : base(index, LoggingSource.Instance.GetLogger <IndexReadOperation>(index._indexStorage.DocumentDatabase.Name)) { try { _analyzer = CreateAnalyzer(() => new LowerCaseKeywordAnalyzer(), index.Definition.IndexFields, forQuerying: true); } catch (Exception e) { throw new IndexAnalyzerException(e); } _queryBuilderFactories = queryBuilderFactories; _maxNumberOfOutputsPerDocument = index.MaxNumberOfOutputsPerDocument; _indexType = index.Type; _indexHasBoostedFields = index.HasBoostedFields; _releaseReadTransaction = directory.SetTransaction(readTransaction, out _state); _releaseSearcher = searcherHolder.GetSearcher(readTransaction, _state, out _searcher); }
public IndexFacetedReadOperation(string indexName, Dictionary <string, IndexField> fields, LuceneVoronDirectory directory, IndexSearcherHolder searcherHolder, Transaction readTransaction, DocumentDatabase documentDatabase) : base(indexName, LoggingSource.Instance.GetLogger <IndexFacetedReadOperation>(documentDatabase.Name)) { try { _analyzer = CreateAnalyzer(() => new LowerCaseKeywordAnalyzer(), fields, forQuerying: true); } catch (Exception e) { throw new IndexAnalyzerException(e); } _releaseReadTransaction = directory.SetTransaction(readTransaction, out _state); _currentStateHolder = searcherHolder.GetStateHolder(readTransaction); _searcher = _currentStateHolder.GetIndexSearcher(_state); }
public IndexWriteOperation(Index index, LuceneVoronDirectory directory, LuceneDocumentConverterBase converter, Transaction writeTransaction, LuceneIndexPersistence persistence) : base(index, LoggingSource.Instance.GetLogger <IndexWriteOperation>(index._indexStorage.DocumentDatabase.Name)) { _converter = converter; DocumentDatabase = index._indexStorage.DocumentDatabase; try { _analyzer = CreateAnalyzer(index, index.Definition); } catch (Exception e) { throw new IndexAnalyzerException(e); } try { _releaseWriteTransaction = directory.SetTransaction(writeTransaction, out _state); _writer = persistence.EnsureIndexWriter(_state); _suggestionsWriters = persistence.EnsureSuggestionIndexWriter(_state); _hasSuggestions = _suggestionsWriters.Count > 0; _locker = directory.MakeLock("writing-to-index.lock"); _directory = directory; if (_locker.Obtain() == false) { throw new InvalidOperationException($"Could not obtain the 'writing-to-index' lock for '{_indexName}' index."); } } catch (Exception e) when(e.IsOutOfMemory()) { throw; } catch (Exception e) { throw new IndexWriteException(e); } }
public IndexFacetedReadOperation(Index index, IndexDefinitionBaseServerSide indexDefinition, LuceneVoronDirectory directory, IndexSearcherHolder searcherHolder, QueryBuilderFactories queryBuilderFactories, Transaction readTransaction, DocumentDatabase documentDatabase) : base(index, LoggingSource.Instance.GetLogger <IndexFacetedReadOperation>(documentDatabase.Name)) { try { _analyzer = CreateAnalyzer(index, indexDefinition, forQuerying: true); } catch (Exception e) { throw new IndexAnalyzerException(e); } _queryBuilderFactories = queryBuilderFactories; _releaseReadTransaction = directory.SetTransaction(readTransaction, out _state); _releaseSearcher = searcherHolder.GetSearcher(readTransaction, _state, out _searcher); }
public MoreLikeThisQueryResult ExecuteMoreLikeThisQuery(MoreLikeThisQuery query, TransactionInformation transactionInformation, int pageSize = 25) { if (query == null) { throw new ArgumentNullException("query"); } var index = database.IndexStorage.GetIndexInstance(query.IndexName); if (index == null) { throw new InvalidOperationException("The index " + query.IndexName + " cannot be found"); } if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0) { throw new InvalidOperationException("The document id or map group fields are mandatory"); } IndexSearcher searcher; using (database.IndexStorage.GetCurrentIndexSearcher(index.indexId, out searcher)) { var documentQuery = new BooleanQuery(); if (string.IsNullOrEmpty(query.DocumentId) == false) { documentQuery.Add(new TermQuery(new Term(Constants.DocumentIdFieldName, query.DocumentId.ToLowerInvariant())), Occur.MUST); } foreach (string key in query.MapGroupFields.Keys) { documentQuery.Add(new TermQuery(new Term(key, query.MapGroupFields[key])), Occur.MUST); } var td = searcher.Search(documentQuery, 1); // get the current Lucene docid for the given RavenDB doc ID if (td.ScoreDocs.Length == 0) { throw new InvalidOperationException("Document " + query.DocumentId + " could not be found"); } var ir = searcher.IndexReader; var mlt = new RavenMoreLikeThis(ir); AssignParameters(mlt, query); if (string.IsNullOrWhiteSpace(query.StopWordsDocumentId) == false) { var stopWordsDoc = database.Documents.Get(query.StopWordsDocumentId, null); if (stopWordsDoc == null) { throw new InvalidOperationException("Stop words document " + query.StopWordsDocumentId + " could not be found"); } var stopWordsSetup = stopWordsDoc.DataAsJson.JsonDeserialization <StopWordsSetup>(); if (stopWordsSetup.StopWords != null) { var stopWords = stopWordsSetup.StopWords; var ht = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase); foreach (var stopWord in stopWords) { ht.Add(stopWord); } mlt.SetStopWords(ht); } } var fieldNames = query.Fields ?? GetFieldNames(ir); mlt.SetFieldNames(fieldNames); var toDispose = new List <Action>(); RavenPerFieldAnalyzerWrapper perFieldAnalyzerWrapper = null; try { perFieldAnalyzerWrapper = index.CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose, true); mlt.Analyzer = perFieldAnalyzerWrapper; var mltQuery = mlt.Like(td.ScoreDocs[0].Doc); var tsdc = TopScoreDocCollector.Create(pageSize, true); if (string.IsNullOrWhiteSpace(query.AdditionalQuery) == false) { var additionalQuery = QueryBuilder.BuildQuery(query.AdditionalQuery, perFieldAnalyzerWrapper); mltQuery = new BooleanQuery { { mltQuery, Occur.MUST }, { additionalQuery, Occur.MUST }, }; } searcher.Search(mltQuery, tsdc); var hits = tsdc.TopDocs().ScoreDocs; var jsonDocuments = GetJsonDocuments(query, searcher, index, query.IndexName, hits, td.ScoreDocs[0].Doc); var result = new MultiLoadResult(); var includedEtags = new List <byte>(jsonDocuments.SelectMany(x => x.Etag.ToByteArray())); includedEtags.AddRange(database.Indexes.GetIndexEtag(query.IndexName, null).ToByteArray()); var loadedIds = new HashSet <string>(jsonDocuments.Select(x => x.Key)); var addIncludesCommand = new AddIncludesCommand(database, transactionInformation, (etag, includedDoc) => { includedEtags.AddRange(etag.ToByteArray()); result.Includes.Add(includedDoc); }, query.Includes ?? new string[0], loadedIds); idsToLoad = new HashSet <string>(); database.TransactionalStorage.Batch(actions => { documentRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, query.TransformerParameters, idsToLoad); using (new CurrentTransformationScope(database, documentRetriever)) { foreach (var document in ProcessResults(query, jsonDocuments, database.WorkContext.CancellationToken)) { result.Results.Add(document); addIncludesCommand.Execute(document); } } }); addIncludesCommand.AlsoInclude(idsToLoad); var computeHash = Encryptor.Current.Hash.Compute16(includedEtags.ToArray()); Etag computedEtag = Etag.Parse(computeHash); return(new MoreLikeThisQueryResult { Etag = computedEtag, Result = result, }); } finally { if (perFieldAnalyzerWrapper != null) { perFieldAnalyzerWrapper.Close(); } foreach (var action in toDispose) { action(); } } } }
protected static RavenPerFieldAnalyzerWrapper CreateAnalyzer(Func <Analyzer> createDefaultAnalyzer, IndexDefinitionBase indexDefinition, bool forQuerying = false) { if (indexDefinition.IndexFields.ContainsKey(Constants.Documents.Indexing.Fields.AllFields)) { throw new InvalidOperationException($"Detected '{Constants.Documents.Indexing.Fields.AllFields}'. This field should not be present here, because inheritance is done elsewhere."); } var hasDefaultFieldOptions = false; Analyzer defaultAnalyzerToUse = null; RavenStandardAnalyzer standardAnalyzer = null; KeywordAnalyzer keywordAnalyzer = null; Analyzer defaultAnalyzer = null; if (indexDefinition is MapIndexDefinition mid) { if (mid.IndexDefinition.Fields.TryGetValue(Constants.Documents.Indexing.Fields.AllFields, out var value)) { hasDefaultFieldOptions = true; switch (value.Indexing) { case FieldIndexing.Exact: defaultAnalyzerToUse = keywordAnalyzer = new KeywordAnalyzer(); break; case FieldIndexing.Search: if (value.Analyzer != null) { defaultAnalyzerToUse = GetAnalyzer(Constants.Documents.Indexing.Fields.AllFields, value.Analyzer, forQuerying); } if (defaultAnalyzerToUse == null) { defaultAnalyzerToUse = standardAnalyzer = new RavenStandardAnalyzer(Version.LUCENE_29); } break; default: // explicitly ignore all other values break; } } } if (defaultAnalyzerToUse == null) { defaultAnalyzerToUse = defaultAnalyzer = createDefaultAnalyzer(); } var perFieldAnalyzerWrapper = new RavenPerFieldAnalyzerWrapper(defaultAnalyzerToUse); foreach (var field in indexDefinition.IndexFields) { var fieldName = field.Value.Name; switch (field.Value.Indexing) { case FieldIndexing.Exact: if (keywordAnalyzer == null) { keywordAnalyzer = new KeywordAnalyzer(); } perFieldAnalyzerWrapper.AddAnalyzer(fieldName, keywordAnalyzer); break; case FieldIndexing.Search: var analyzer = GetAnalyzer(fieldName, field.Value.Analyzer, forQuerying); if (analyzer != null) { perFieldAnalyzerWrapper.AddAnalyzer(fieldName, analyzer); continue; } AddStandardAnalyzer(fieldName); break; case FieldIndexing.Default: if (hasDefaultFieldOptions) { // if we have default field options then we need to take into account overrides for regular fields if (defaultAnalyzer == null) { defaultAnalyzer = createDefaultAnalyzer(); } perFieldAnalyzerWrapper.AddAnalyzer(fieldName, defaultAnalyzer); continue; } break; } } return(perFieldAnalyzerWrapper); void AddStandardAnalyzer(string fieldName) { if (standardAnalyzer == null) { standardAnalyzer = new RavenStandardAnalyzer(Version.LUCENE_29); } perFieldAnalyzerWrapper.AddAnalyzer(fieldName, standardAnalyzer); } }
private static Query Parse(string q) { using (var defaultAnalyzer = new KeywordAnalyzer()) using (var perFieldAnalyzerWrapper = new RavenPerFieldAnalyzerWrapper(defaultAnalyzer)) return(QueryBuilder.BuildQuery(q, perFieldAnalyzerWrapper)); }
public SimpleFilesQueryParser(Analyzer analyzer) : base(Version.LUCENE_29, string.Empty, analyzer) { fieldAnalyzer = new RavenPerFieldAnalyzerWrapper(fieldAnalyzer); }
public MoreLikeThisQueryResult ExecuteMoreLikeThisQuery(MoreLikeThisQuery query, TransactionInformation transactionInformation, int pageSize = 25, string[] include = null) { if (query == null) { throw new ArgumentNullException("query"); } var index = database.IndexStorage.GetIndexInstance(query.IndexName); if (index == null) { throw new InvalidOperationException("The index " + query.IndexName + " cannot be found"); } if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0) { throw new InvalidOperationException("The document id or map group fields are mandatory"); } IndexSearcher searcher; using (database.IndexStorage.GetCurrentIndexSearcher(query.IndexName, out searcher)) { var documentQuery = new BooleanQuery(); if (string.IsNullOrEmpty(query.DocumentId) == false) { documentQuery.Add(new TermQuery(new Term(Constants.DocumentIdFieldName, query.DocumentId.ToLowerInvariant())), Occur.MUST); } foreach (string key in query.MapGroupFields.Keys) { documentQuery.Add(new TermQuery(new Term(key, query.MapGroupFields[key])), Occur.MUST); } var td = searcher.Search(documentQuery, 1); // get the current Lucene docid for the given RavenDB doc ID if (td.ScoreDocs.Length == 0) { throw new InvalidOperationException("Document " + query.DocumentId + " could not be found"); } var ir = searcher.IndexReader; var mlt = new RavenMoreLikeThis(ir); AssignParameters(mlt, query); if (string.IsNullOrWhiteSpace(query.StopWordsDocumentId) == false) { var stopWordsDoc = database.Get(query.StopWordsDocumentId, null); if (stopWordsDoc == null) { throw new InvalidOperationException("Stop words document " + query.StopWordsDocumentId + " could not be found"); } var stopWordsSetup = stopWordsDoc.DataAsJson.JsonDeserialization <StopWordsSetup>(); if (stopWordsSetup.StopWords != null) { var stopWords = stopWordsSetup.StopWords; var ht = new Hashtable(StringComparer.InvariantCultureIgnoreCase); foreach (var stopWord in stopWords) { ht[stopWord] = stopWord; } mlt.SetStopWords(ht); } } var fieldNames = query.Fields ?? GetFieldNames(ir); mlt.SetFieldNames(fieldNames); var toDispose = new List <Action>(); RavenPerFieldAnalyzerWrapper perFieldAnalyzerWrapper = null; try { perFieldAnalyzerWrapper = index.CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose, true); mlt.SetAnalyzer(perFieldAnalyzerWrapper); var mltQuery = mlt.Like(td.ScoreDocs[0].Doc); var tsdc = TopScoreDocCollector.Create(pageSize, true); searcher.Search(mltQuery, tsdc); var hits = tsdc.TopDocs().ScoreDocs; var jsonDocuments = GetJsonDocuments(query, searcher, index, query.IndexName, hits, td.ScoreDocs[0].Doc); var result = new MultiLoadResult(); var includedEtags = new List <byte>(jsonDocuments.SelectMany(x => x.Etag.Value.ToByteArray())); includedEtags.AddRange(database.GetIndexEtag(query.IndexName, null).ToByteArray()); var loadedIds = new HashSet <string>(jsonDocuments.Select(x => x.Key)); var addIncludesCommand = new AddIncludesCommand(database, transactionInformation, (etag, includedDoc) => { includedEtags.AddRange(etag.ToByteArray()); result.Includes.Add(includedDoc); }, include ?? new string[0], loadedIds); foreach (var jsonDocument in jsonDocuments) { result.Results.Add(jsonDocument.ToJson()); addIncludesCommand.Execute(jsonDocument.DataAsJson); } Guid computedEtag; using (var md5 = MD5.Create()) { var computeHash = md5.ComputeHash(includedEtags.ToArray()); computedEtag = new Guid(computeHash); } return(new MoreLikeThisQueryResult { Etag = computedEtag, Result = result, }); } finally { if (perFieldAnalyzerWrapper != null) { perFieldAnalyzerWrapper.Close(); } foreach (var action in toDispose) { action(); } } } }