public static void ReadEntriesForFieldsFromTermVectors( IndexSearcherHolder.IndexSearcherHoldingState state, HashSet<string> fieldsToRead, HashSet<int> docIds, Func<string,string, double> convert, Action<string, string, double, int> onTermFound) { var reader = state.IndexSearcher.IndexReader; foreach (var docId in docIds) { foreach (var field in fieldsToRead) { var termFreqVector = reader.GetTermFreqVector(docId, field); if (termFreqVector == null) continue; foreach (var term in termFreqVector.GetTerms()) { if (LowPrecisionNumber(field, term)) continue; onTermFound(field, term, convert(field, term), docId); } } } }
public static void ReadEntriesForFields( IndexSearcherHolder.IndexSearcherHoldingState state, HashSet<string> fieldsToRead, HashSet<int> docIds, Action<Term, int> onTermFound) { var reader = state.IndexSearcher.IndexReader; state.Lock.EnterReadLock(); try { EnsureFieldsAreInCache(state, fieldsToRead, reader); foreach (var field in fieldsToRead) { foreach (var docId in docIds) { foreach (var term in state.GetTermsFromCache(field, docId)) { onTermFound(term, docId); } } } } finally { if (state.Lock.IsReadLockHeld) state.Lock.ExitReadLock(); } }
protected Index(Directory directory, string name, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context) { currentIndexSearcherHolder = new IndexSearcherHolder(name ,context); if (directory == null) throw new ArgumentNullException("directory"); if (name == null) throw new ArgumentNullException("name"); if (indexDefinition == null) throw new ArgumentNullException("indexDefinition"); if (viewGenerator == null) throw new ArgumentNullException("viewGenerator"); this.name = name; this.indexDefinition = indexDefinition; this.viewGenerator = viewGenerator; this.context = context; logIndexing.Debug("Creating index for {0}", name); this.directory = directory; RecreateSearcher(); }
public static void ReadEntriesForFields( IndexSearcherHolder.IndexSearcherHoldingState state, HashSet<string> fieldsToRead, HashSet<int> docIds, Func<Term, double> convert, Action<Term, double, int> onTermFound) { var reader = state.IndexSearcher.IndexReader; var readFromCache = new Dictionary<string, HashSet<int>>(); state.Lock.EnterReadLock(); try { EnsureFieldsAreInCache(state, fieldsToRead, reader); foreach (var field in fieldsToRead) { var read = new HashSet<int>(); readFromCache[field] = read; foreach (var docId in docIds) { foreach (var val in state.GetFromCache(field, docId)) { read.Add(docId); double converted; if (val.Val == null) { val.Val = converted = convert(val.Term); } else { converted = val.Val.Value; } onTermFound(val.Term, converted, docId); } } } } finally { if (state.Lock.IsReadLockHeld) state.Lock.ExitReadLock(); } }
private void RecreateSearcher() { var oldSearcher = currentIndexSearcherHolder; if (indexWriter == null) { currentIndexSearcherHolder = new IndexSearcherHolder(new IndexSearcher(directory, true)); } else { var indexReader = indexWriter.GetReader(); currentIndexSearcherHolder = new IndexSearcherHolder(new IndexSearcher(indexReader)); } if (oldSearcher != null) { IndexSearcher _; using (oldSearcher.GetSearcher(out _)) { oldSearcher.DisposeSafely(); } } }
public static void ReadEntriesForFields( IndexSearcherHolder.IndexSearcherHoldingState state, HashSet<string> fieldsToRead, HashSet<int> docIds, Func<Term, double> convert, Action<Term, double, int> onTermFound) { var reader = state.IndexSearcher.IndexReader; var readFromCache = new Dictionary<string, HashSet<int>>(); state.Lock.EnterReadLock(); try { foreach (var field in fieldsToRead) { var read = new HashSet<int>(); readFromCache[field] = read; foreach (var docId in docIds) { foreach (var val in state.GetFromCache(field, docId)) { read.Add(docId); double converted; if (val.Val == null) { val.Val = converted = convert(val.Term); } else { converted = val.Val.Value; } onTermFound(val.Term, converted, docId); } } } } finally { state.Lock.ExitReadLock(); } foreach (var kvp in readFromCache) { if (kvp.Value.Count == docIds.Count) { fieldsToRead.Remove(kvp.Key); // already read all of it } } if (fieldsToRead.Count == 0) return; state.Lock.EnterWriteLock(); try { using (var termDocs = reader.TermDocs()) { foreach (var field in fieldsToRead) { var read = readFromCache[field]; var shouldReset = new HashSet<Tuple<string, int>>(); using (var termEnum = reader.Terms(new Term(field))) { do { if (termEnum.Term == null || field != termEnum.Term.Field) break; if (LowPrecisionNumber(termEnum.Term)) continue; var totalDocCountIncludedDeletes = termEnum.DocFreq(); termDocs.Seek(termEnum.Term); while (termDocs.Next() && totalDocCountIncludedDeletes > 0) { totalDocCountIncludedDeletes -= 1; if (read.Contains(termDocs.Doc)) continue; if (reader.IsDeleted(termDocs.Doc)) continue; if (docIds.Contains(termDocs.Doc) == false) continue; if (shouldReset.Add(Tuple.Create(field, termDocs.Doc))) { state.ResetInCache(field, termDocs.Doc); } var d = convert(termEnum.Term); state.SetInCache(field, termDocs.Doc, termEnum.Term, d); onTermFound(termEnum.Term, d, termDocs.Doc); } } while (termEnum.Next()); } } } } finally { state.Lock.ExitWriteLock(); } }
private void CompleteFacetCalculationsStage1(IndexSearcherHolder.IndexSearcherHoldingState state) { var fieldsToRead = new HashSet<string>(Facets .Where(x => x.Value.Aggregation != FacetAggregation.None && x.Value.Aggregation != FacetAggregation.Count) .Select(x => x.Value.AggregationField) .Where(x => x != null)); if (fieldsToRead.Count == 0) return; var allDocs = new HashSet<int>(matches.Values.SelectMany(x => x.Docs)); IndexedTerms.ReadEntriesForFields(state, fieldsToRead, allDocs, GetValueFromIndex, (term, currentVal, docId) => { foreach (var match in matches) { if (match.Value.Docs.Contains(docId) == false) continue; var facet = match.Value.Facet; if (term.Field != facet.AggregationField) continue; switch (facet.Mode) { case FacetMode.Default: ApplyAggregation(facet, match.Key, currentVal); break; case FacetMode.Ranges: if (!match.Value.Range.IsMatch(term.Text)) continue; ApplyAggregation(facet, match.Key, currentVal); break; default: throw new ArgumentOutOfRangeException(); } } }); }
private void CompleteFacetCalculationsStage1(IndexSearcherHolder.IndexSearcherHoldingState state, bool allVectoredTerms) { var fieldsToRead = new HashSet<string>(Facets .Where(x => x.Value.Aggregation != FacetAggregation.None && x.Value.Aggregation != FacetAggregation.Count) .Select(x => x.Value.AggregationField) .Where(x => x != null)); if (fieldsToRead.Count == 0) return; var allDocs = new HashSet<int>(matches.Values.SelectMany(x => x.Docs)); if (allVectoredTerms) { IndexedTerms.ReadEntriesForFieldsFromTermVectors(state, fieldsToRead, allDocs, GetValueFromIndex, (field, textVal, currentVal, docId) => HandleFacetsCalculationStage1(docId, field, textVal, currentVal)); } else { IndexedTerms.ReadEntriesForFields(state, fieldsToRead, allDocs, GetValueFromIndex, (field, textVal, currentVal, docId) => HandleFacetsCalculationStage1(docId, field, textVal, currentVal)); } }
private void RecreateSearcher() { var oldSearcher = currentIndexSearcherHolder; if (indexWriter == null) { currentIndexSearcherHolder = new IndexSearcherHolder(new IndexSearcher(directory, true)); } else { var indexReader = indexWriter.GetReader(); currentIndexSearcherHolder = new IndexSearcherHolder(new IndexSearcher(indexReader)); } if (oldSearcher != null) { IndexSearcher _; using(oldSearcher.GetSearcher(out _)) { oldSearcher.DisposeSafely(); } } }
private static void FillCache(IndexSearcherHolder.IndexSearcherHoldingState state, IEnumerable<string> fieldsToRead,IndexReader reader) { foreach (var field in fieldsToRead) { var items = new LinkedList<IndexSearcherHolder.IndexSearcherHoldingState.CacheVal>[reader.MaxDoc]; using (var termDocs = reader.TermDocs()) { using (var termEnum = reader.Terms(new Term(field))) { do { if (termEnum.Term == null || field != termEnum.Term.Field) break; Term term = termEnum.Term; if (LowPrecisionNumber(term.Field, term.Text)) continue; var totalDocCountIncludedDeletes = termEnum.DocFreq(); termDocs.Seek(termEnum.Term); while (termDocs.Next() && totalDocCountIncludedDeletes > 0) { totalDocCountIncludedDeletes -= 1; if (reader.IsDeleted(termDocs.Doc)) continue; if(items[termDocs.Doc] == null) items[termDocs.Doc] = new LinkedList<IndexSearcherHolder.IndexSearcherHoldingState.CacheVal>(); items[termDocs.Doc].AddLast(new IndexSearcherHolder.IndexSearcherHoldingState.CacheVal { Term = termEnum.Term }); } } while (termEnum.Next()); } } state.SetInCache(field, items); } }
private static void EnsureFieldsAreInCache(IndexSearcherHolder.IndexSearcherHoldingState state, HashSet<string> fieldsToRead, IndexReader reader) { if (fieldsToRead.All(state.IsInCache)) return; state.Lock.ExitReadLock(); state.Lock.EnterWriteLock(); try { var fieldsNotInCache = fieldsToRead.Where(field => state.IsInCache(field) == false).ToList(); if (fieldsToRead.Count > 0) FillCache(state, fieldsNotInCache, reader); } finally { state.Lock.ExitWriteLock(); } state.Lock.EnterReadLock(); }
public static void PreFillCache(IndexSearcherHolder.IndexSearcherHoldingState state, string[] fieldsToRead, IndexReader reader) { state.Lock.EnterWriteLock(); try { if (fieldsToRead.All(state.IsInCache)) return; FillCache(state, fieldsToRead, reader); } finally { state.Lock.ExitWriteLock(); } }
private static void FillCache(IndexSearcherHolder.IndexSearcherHoldingState state, List<string> fieldsToRead,IndexReader reader) { foreach (var field in fieldsToRead) { using (var termDocs = reader.TermDocs()) { using (var termEnum = reader.Terms(new Term(field))) { do { if (termEnum.Term == null || field != termEnum.Term.Field) break; if (LowPrecisionNumber(termEnum.Term)) continue; var totalDocCountIncludedDeletes = termEnum.DocFreq(); termDocs.Seek(termEnum.Term); while (termDocs.Next() && totalDocCountIncludedDeletes > 0) { totalDocCountIncludedDeletes -= 1; if (reader.IsDeleted(termDocs.Doc)) continue; state.SetInCache(field, termDocs.Doc, termEnum.Term); } } while (termEnum.Next()); } } } }