private static void AssertAnalyzersValid(IndexDefinition indexDefinition) { foreach (var analyzer in indexDefinition.Analyzers) { IndexingExtensions.CreateAnalyzerInstance(analyzer.Key, analyzer.Value); } }
private static void AssertAnalyzersValid(IndexDefinition indexDefinition) { foreach (var analyzer in indexDefinition.Analyzers) { //this throws if the type cannot be found IndexingExtensions.GetAnalyzerType(analyzer.Key, analyzer.Value); } }
public PerFieldAnalyzerWrapper CreateAnalyzer(Analyzer defaultAnalyzer, ICollection <Action> toDispose, bool forQuerying = false) { toDispose.Add(defaultAnalyzer.Close); var perFieldAnalyzerWrapper = new PerFieldAnalyzerWrapper(defaultAnalyzer); foreach (var analyzer in indexDefinition.Analyzers) { Analyzer analyzerInstance = IndexingExtensions.CreateAnalyzerInstance(analyzer.Key, analyzer.Value); if (analyzerInstance == null) { continue; } toDispose.Add(analyzerInstance.Close); if (forQuerying) { var customAttributes = analyzerInstance.GetType().GetCustomAttributes(typeof(NotForQueryingAttribute), false); if (customAttributes.Length > 0) { continue; } } perFieldAnalyzerWrapper.AddAnalyzer(analyzer.Key, analyzerInstance); } StandardAnalyzer standardAnalyzer = null; KeywordAnalyzer keywordAnalyzer = null; foreach (var fieldIndexing in indexDefinition.Indexes) { switch (fieldIndexing.Value) { case FieldIndexing.NotAnalyzed: if (keywordAnalyzer == null) { keywordAnalyzer = new KeywordAnalyzer(); toDispose.Add(keywordAnalyzer.Close); } perFieldAnalyzerWrapper.AddAnalyzer(fieldIndexing.Key, keywordAnalyzer); break; case FieldIndexing.Analyzed: if (indexDefinition.Analyzers.ContainsKey(fieldIndexing.Key)) { continue; } if (standardAnalyzer == null) { standardAnalyzer = new StandardAnalyzer(Version.LUCENE_29); toDispose.Add(standardAnalyzer.Close); } perFieldAnalyzerWrapper.AddAnalyzer(fieldIndexing.Key, standardAnalyzer); break; } } return(perFieldAnalyzerWrapper); }
public PerFieldAnalyzerWrapper CreateAnalyzer(Analyzer defaultAnalyzer, ICollection <Action> toDispose) { toDispose.Add(defaultAnalyzer.Close); var perFieldAnalyzerWrapper = new PerFieldAnalyzerWrapper(defaultAnalyzer); foreach (var analyzer in indexDefinition.Analyzers) { var analyzerInstance = IndexingExtensions.CreateAnalyzerInstance(analyzer.Key, analyzer.Value); if (analyzerInstance == null) { continue; } toDispose.Add(analyzerInstance.Close); perFieldAnalyzerWrapper.AddAnalyzer(analyzer.Key, analyzerInstance); } StandardAnalyzer standardAnalyzer = null; KeywordAnalyzer keywordAnalyzer = null; foreach (var fieldIndexing in indexDefinition.Indexes) { switch (fieldIndexing.Value) { case FieldIndexing.NotAnalyzed: case FieldIndexing.NotAnalyzedNoNorms: if (keywordAnalyzer == null) { keywordAnalyzer = new KeywordAnalyzer(); toDispose.Add(keywordAnalyzer.Close); } perFieldAnalyzerWrapper.AddAnalyzer(fieldIndexing.Key, keywordAnalyzer); break; case FieldIndexing.Analyzed: if (indexDefinition.Analyzers.ContainsKey(fieldIndexing.Key)) { continue; } if (standardAnalyzer == null) { standardAnalyzer = new StandardAnalyzer(Version.LUCENE_29); toDispose.Add(standardAnalyzer.Close); } perFieldAnalyzerWrapper.AddAnalyzer(fieldIndexing.Key, standardAnalyzer); break; } } return(perFieldAnalyzerWrapper); }
private static void ValidateAnalyzers(IndexDefinition definition) { if (definition.Fields == null) { return; } foreach (var kvp in definition.Fields) { if (string.IsNullOrWhiteSpace(kvp.Value.Analyzer)) { continue; } try { IndexingExtensions.GetAnalyzerType(kvp.Key, kvp.Value.Analyzer); } catch (Exception e) { throw new IndexCompilationException(e.Message, e); } } }
public MoreLikeThisQueryResult ExecuteMoreLikeThisQuery(MoreLikeThisQuery query, TransactionInformation transactionInformation, int pageSize = 25) { if (query == null) { throw new ArgumentNullException("query"); } var index = database.IndexStorage.GetIndexInstance(query.IndexName); if (index == null) { throw new InvalidOperationException("The index " + query.IndexName + " cannot be found"); } if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0 && string.IsNullOrEmpty(query.Document)) { throw new InvalidOperationException("The document id, map group fields or document are mandatory"); } IndexSearcher searcher; using (database.IndexStorage.GetCurrentIndexSearcher(index.indexId, out searcher)) { int?baseDocId = null; if (string.IsNullOrEmpty(query.DocumentId) == false || query.MapGroupFields.Count > 0) { var documentQuery = new BooleanQuery(); if (string.IsNullOrEmpty(query.DocumentId) == false) { documentQuery.Add(new TermQuery(new Term(Constants.DocumentIdFieldName, query.DocumentId.ToLowerInvariant())), Occur.MUST); } foreach (string key in query.MapGroupFields.Keys) { documentQuery.Add(new TermQuery(new Term(key, query.MapGroupFields[key])), Occur.MUST); } var td = searcher.Search(documentQuery, 1); // get the current Lucene docid for the given RavenDB doc ID if (td.ScoreDocs.Length == 0) { throw new InvalidOperationException("Document " + query.DocumentId + " could not be found"); } baseDocId = td.ScoreDocs[0].Doc; } var ir = searcher.IndexReader; var mlt = new RavenMoreLikeThis(ir); AssignParameters(mlt, query); if (string.IsNullOrWhiteSpace(query.StopWordsDocumentId) == false) { var stopWordsDoc = database.Documents.Get(query.StopWordsDocumentId, null); if (stopWordsDoc == null) { throw new InvalidOperationException("Stop words document " + query.StopWordsDocumentId + " could not be found"); } var stopWordsSetup = stopWordsDoc.DataAsJson.JsonDeserialization <StopWordsSetup>(); if (stopWordsSetup.StopWords != null) { var stopWords = stopWordsSetup.StopWords; var ht = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase); foreach (var stopWord in stopWords) { ht.Add(stopWord); } mlt.SetStopWords(ht); } } var fieldNames = query.Fields ?? GetFieldNames(ir); mlt.SetFieldNames(fieldNames); var toDispose = new List <Action>(); RavenPerFieldAnalyzerWrapper perFieldAnalyzerWrapper = null; try { var defaultAnalyzer = !string.IsNullOrWhiteSpace(query.DefaultAnalyzerName) ? IndexingExtensions.CreateAnalyzerInstance(Constants.AllFields, query.DefaultAnalyzerName) : new LowerCaseKeywordAnalyzer(); perFieldAnalyzerWrapper = index.CreateAnalyzer(defaultAnalyzer, toDispose, true); mlt.Analyzer = perFieldAnalyzerWrapper; var mltQuery = baseDocId.HasValue ? mlt.Like(baseDocId.Value) : mlt.Like(RavenJObject.Parse(query.Document)); var tsdc = TopScoreDocCollector.Create(pageSize, true); if (string.IsNullOrWhiteSpace(query.AdditionalQuery) == false) { var additionalQuery = QueryBuilder.BuildQuery(query.AdditionalQuery, perFieldAnalyzerWrapper); mltQuery = new BooleanQuery { { mltQuery, Occur.MUST }, { additionalQuery, Occur.MUST }, }; } searcher.Search(mltQuery, tsdc); var hits = tsdc.TopDocs().ScoreDocs; var jsonDocuments = baseDocId.HasValue ? GetJsonDocuments(query, searcher, index, query.IndexName, hits, baseDocId.Value) : GetJsonDocuments(searcher, index, query.IndexName, hits); var result = new MultiLoadResult(); var includedEtags = new List <byte>(jsonDocuments.SelectMany(x => x.Etag.ToByteArray())); includedEtags.AddRange(database.Indexes.GetIndexEtag(query.IndexName, null).ToByteArray()); var loadedIds = new HashSet <string>(jsonDocuments.Select(x => x.Key)); var addIncludesCommand = new AddIncludesCommand(database, transactionInformation, (etag, includedDoc) => { includedEtags.AddRange(etag.ToByteArray()); result.Includes.Add(includedDoc); }, query.Includes ?? new string[0], loadedIds); idsToLoad = new HashSet <string>(); database.TransactionalStorage.Batch(actions => { documentRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, query.TransformerParameters, idsToLoad); using (new CurrentTransformationScope(database, documentRetriever)) { foreach (var document in ProcessResults(query, jsonDocuments, database.WorkContext.CancellationToken)) { result.Results.Add(document); addIncludesCommand.Execute(document); } } }); addIncludesCommand.AlsoInclude(idsToLoad); var computeHash = Encryptor.Current.Hash.Compute16(includedEtags.ToArray()); Etag computedEtag = Etag.Parse(computeHash); return(new MoreLikeThisQueryResult { Etag = computedEtag, Result = result, }); } finally { if (perFieldAnalyzerWrapper != null) { perFieldAnalyzerWrapper.Close(); } foreach (var action in toDispose) { action(); } } } }
public virtual Analyzer CreateInstance(string fieldName) { return(IndexingExtensions.CreateAnalyzerInstance(fieldName, Type)); }