public static void ReadEntriesForFieldsFromTermVectors(
            IndexSearcherHolder.IndexSearcherHoldingState state,
            HashSet<string> fieldsToRead,
            HashSet<int> docIds,
            Func<string,string, double> convert,
            Action<string, string, double, int> onTermFound)
        {
            var reader = state.IndexSearcher.IndexReader;

            foreach (var docId in docIds)
            {
                foreach (var field in fieldsToRead)
                {
                    var termFreqVector = reader.GetTermFreqVector(docId, field);
                    if (termFreqVector == null)
                        continue;

                    foreach (var term in termFreqVector.GetTerms())
                    {
                        if (LowPrecisionNumber(field, term))
                            continue;
                        onTermFound(field, term, convert(field, term), docId);
                    }
                }
            }
        }
        public static void ReadEntriesForFields(
              IndexSearcherHolder.IndexSearcherHoldingState state,
              HashSet<string> fieldsToRead,
              HashSet<int> docIds,
              Action<Term, int> onTermFound)
        {
            var reader = state.IndexSearcher.IndexReader;

            state.Lock.EnterReadLock();
            try
            {
                EnsureFieldsAreInCache(state, fieldsToRead, reader);

                foreach (var field in fieldsToRead)
                {
                    foreach (var docId in docIds)
                    {
                        foreach (var term in state.GetTermsFromCache(field, docId))
                        {
                            onTermFound(term, docId);
                        }
                    }
                }
            }
            finally
            {
                if (state.Lock.IsReadLockHeld)
                    state.Lock.ExitReadLock();
            }
        }
Beispiel #3
0
		protected Index(Directory directory, string name, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context)
		{
		    currentIndexSearcherHolder = new IndexSearcherHolder(name ,context);
		    if (directory == null) throw new ArgumentNullException("directory");
			if (name == null) throw new ArgumentNullException("name");
			if (indexDefinition == null) throw new ArgumentNullException("indexDefinition");
			if (viewGenerator == null) throw new ArgumentNullException("viewGenerator");

			this.name = name;
			this.indexDefinition = indexDefinition;
			this.viewGenerator = viewGenerator;
			this.context = context;
			logIndexing.Debug("Creating index for {0}", name);
			this.directory = directory;

			RecreateSearcher();
		}
        public static void ReadEntriesForFields(
                IndexSearcherHolder.IndexSearcherHoldingState state,
                HashSet<string> fieldsToRead,
                HashSet<int> docIds,
                Func<Term, double> convert,
                Action<Term, double, int> onTermFound)
        {
            var reader = state.IndexSearcher.IndexReader;

            var readFromCache = new Dictionary<string, HashSet<int>>();

            state.Lock.EnterReadLock();
            try
            {
                EnsureFieldsAreInCache(state, fieldsToRead, reader);

                foreach (var field in fieldsToRead)
                {
                    var read = new HashSet<int>();
                    readFromCache[field] = read;
                    foreach (var docId in docIds)
                    {
                        foreach (var val in state.GetFromCache(field, docId))
                        {
                            read.Add(docId);

                            double converted;
                            if (val.Val == null)
                            {
                                val.Val = converted = convert(val.Term);
                            }
                            else
                            {
                                converted = val.Val.Value;
                            }
                            onTermFound(val.Term, converted, docId);
                        }
                    }
                }
            }
            finally
            {
                if (state.Lock.IsReadLockHeld)
                    state.Lock.ExitReadLock();
            }
        }
Beispiel #5
0
        private void RecreateSearcher()
        {
            var oldSearcher = currentIndexSearcherHolder;

            if (indexWriter == null)
            {
                currentIndexSearcherHolder = new IndexSearcherHolder(new IndexSearcher(directory, true));
            }
            else
            {
                var indexReader = indexWriter.GetReader();
                currentIndexSearcherHolder = new IndexSearcherHolder(new IndexSearcher(indexReader));
            }

            if (oldSearcher != null)
            {
                IndexSearcher _;
                using (oldSearcher.GetSearcher(out _))
                {
                    oldSearcher.DisposeSafely();
                }
            }
        }
Beispiel #6
0
        public static void ReadEntriesForFields(
                IndexSearcherHolder.IndexSearcherHoldingState state,
                HashSet<string> fieldsToRead,
                HashSet<int> docIds,
                Func<Term, double> convert,
                Action<Term, double, int> onTermFound)
        {
            var reader = state.IndexSearcher.IndexReader;

            var readFromCache = new Dictionary<string, HashSet<int>>();

            state.Lock.EnterReadLock();
            try
            {
                foreach (var field in fieldsToRead)
                {
                    var read = new HashSet<int>();
                    readFromCache[field] = read;
                    foreach (var docId in docIds)
                    {
                        foreach (var val in state.GetFromCache(field, docId))
                        {
                            read.Add(docId);

                            double converted;
                            if (val.Val == null)
                            {
                                val.Val = converted = convert(val.Term);
                            }
                            else
                            {
                                converted = val.Val.Value;
                            }
                            onTermFound(val.Term, converted, docId);
                        }
                    }
                }
            }
            finally
            {
                state.Lock.ExitReadLock();
            }


            foreach (var kvp in readFromCache)
            {
                if (kvp.Value.Count == docIds.Count)
                {
                    fieldsToRead.Remove(kvp.Key); // already read all of it
                }
            }

            if (fieldsToRead.Count == 0)
                return;
            
            state.Lock.EnterWriteLock();
            try
            {
                using (var termDocs = reader.TermDocs())
                {
                    foreach (var field in fieldsToRead)
                    {
                        var read = readFromCache[field];
						var shouldReset = new HashSet<Tuple<string, int>>();
                        using (var termEnum = reader.Terms(new Term(field)))
                        {
                            do
                            {
                                if (termEnum.Term == null || field != termEnum.Term.Field)
                                    break;

                                if (LowPrecisionNumber(termEnum.Term))
                                    continue;

                                    
                                var totalDocCountIncludedDeletes = termEnum.DocFreq();
                                termDocs.Seek(termEnum.Term);

                                while (termDocs.Next() && totalDocCountIncludedDeletes > 0)
                                {
                                    totalDocCountIncludedDeletes -= 1;
                                    if (read.Contains(termDocs.Doc))
										continue;
                                    if (reader.IsDeleted(termDocs.Doc))
										continue;
                                    if (docIds.Contains(termDocs.Doc) == false)
										continue;

									if (shouldReset.Add(Tuple.Create(field, termDocs.Doc)))
									{
										state.ResetInCache(field, termDocs.Doc);
									}

                                    var d = convert(termEnum.Term);
                                    state.SetInCache(field, termDocs.Doc, termEnum.Term, d);
                                    onTermFound(termEnum.Term, d, termDocs.Doc);
                                }
                            } while (termEnum.Next());
                        }
                    }
                }
            }
            finally
            {
                state.Lock.ExitWriteLock();
            }
        }
Beispiel #7
0
            private void CompleteFacetCalculationsStage1(IndexSearcherHolder.IndexSearcherHoldingState state)
            {
                var fieldsToRead = new HashSet<string>(Facets
                        .Where(x => x.Value.Aggregation != FacetAggregation.None && x.Value.Aggregation != FacetAggregation.Count)
                        .Select(x => x.Value.AggregationField)
                        .Where(x => x != null));

                if (fieldsToRead.Count == 0)
                    return;

                var allDocs = new HashSet<int>(matches.Values.SelectMany(x => x.Docs));

                IndexedTerms.ReadEntriesForFields(state, fieldsToRead, allDocs, GetValueFromIndex, (term, currentVal, docId) =>
                {
                    foreach (var match in matches)
                    {
                        if (match.Value.Docs.Contains(docId) == false)
                            continue;
                        var facet = match.Value.Facet;
                        if (term.Field != facet.AggregationField)
                            continue;
                        switch (facet.Mode)
                        {
                            case FacetMode.Default:
                                ApplyAggregation(facet, match.Key, currentVal);
                                break;
                            case FacetMode.Ranges:
                                if (!match.Value.Range.IsMatch(term.Text))
                                    continue;
                                ApplyAggregation(facet, match.Key, currentVal);
                                break;
                            default:
                                throw new ArgumentOutOfRangeException();
                        }
                    }
                });
            }
Beispiel #8
0
            private void CompleteFacetCalculationsStage1(IndexSearcherHolder.IndexSearcherHoldingState state, bool allVectoredTerms)
            {
                var fieldsToRead = new HashSet<string>(Facets
                        .Where(x => x.Value.Aggregation != FacetAggregation.None && x.Value.Aggregation != FacetAggregation.Count)
                        .Select(x => x.Value.AggregationField)
                        .Where(x => x != null));

                if (fieldsToRead.Count == 0)
                    return;

                var allDocs = new HashSet<int>(matches.Values.SelectMany(x => x.Docs));

                if (allVectoredTerms)
                {
                    IndexedTerms.ReadEntriesForFieldsFromTermVectors(state, fieldsToRead, allDocs, GetValueFromIndex,
                        (field, textVal, currentVal, docId) =>
                            HandleFacetsCalculationStage1(docId, field, textVal, currentVal));
                }
                else
                {
                    IndexedTerms.ReadEntriesForFields(state, fieldsToRead, allDocs, GetValueFromIndex,
                        (field, textVal, currentVal, docId) =>
                            HandleFacetsCalculationStage1(docId, field, textVal, currentVal));
                }
            }
Beispiel #9
0
		private void RecreateSearcher()
		{
			var oldSearcher = currentIndexSearcherHolder;
            
		    if (indexWriter == null)
		    {
		    	currentIndexSearcherHolder = new IndexSearcherHolder(new IndexSearcher(directory, true));
		    }
		    else
		    {
		        var indexReader = indexWriter.GetReader();
				currentIndexSearcherHolder = new IndexSearcherHolder(new IndexSearcher(indexReader));
		    }

            if (oldSearcher != null)
            {
            	IndexSearcher _;
            	using(oldSearcher.GetSearcher(out _))
            	{
            		oldSearcher.DisposeSafely();
            	}
            }
		}
Beispiel #10
0
        private static void FillCache(IndexSearcherHolder.IndexSearcherHoldingState state, IEnumerable<string> fieldsToRead,IndexReader reader)
        {
            foreach (var field in fieldsToRead)
            {
	            var items = new LinkedList<IndexSearcherHolder.IndexSearcherHoldingState.CacheVal>[reader.MaxDoc];
                using (var termDocs = reader.TermDocs())
                {
                    using (var termEnum = reader.Terms(new Term(field)))
                    {
                        do
                        {
                            if (termEnum.Term == null || field != termEnum.Term.Field)
                                break;

                            Term term = termEnum.Term;
                            if (LowPrecisionNumber(term.Field, term.Text))
                                continue;


                            var totalDocCountIncludedDeletes = termEnum.DocFreq();
                            termDocs.Seek(termEnum.Term);

                            while (termDocs.Next() && totalDocCountIncludedDeletes > 0)
                            {
                                totalDocCountIncludedDeletes -= 1;
                                if (reader.IsDeleted(termDocs.Doc))
                                    continue;
								if(items[termDocs.Doc] == null)
									items[termDocs.Doc] = new LinkedList<IndexSearcherHolder.IndexSearcherHoldingState.CacheVal>();

	                            items[termDocs.Doc].AddLast(new IndexSearcherHolder.IndexSearcherHoldingState.CacheVal
	                            {
		                            Term = termEnum.Term
	                            });
                            }
                        } while (termEnum.Next());
                    }
                }
	            state.SetInCache(field, items);
            }
        }
Beispiel #11
0
        private static void EnsureFieldsAreInCache(IndexSearcherHolder.IndexSearcherHoldingState state, HashSet<string> fieldsToRead, IndexReader reader)
        {
            if (fieldsToRead.All(state.IsInCache))
                return;

            state.Lock.ExitReadLock();
            state.Lock.EnterWriteLock();
            try
            {
                var fieldsNotInCache = fieldsToRead.Where(field => state.IsInCache(field) == false).ToList();
                if (fieldsToRead.Count > 0)
                    FillCache(state, fieldsNotInCache, reader);
            }
            finally
            {
                state.Lock.ExitWriteLock();
            }
            state.Lock.EnterReadLock();
        }
Beispiel #12
0
 public static void PreFillCache(IndexSearcherHolder.IndexSearcherHoldingState state, string[] fieldsToRead,
     IndexReader reader)
 {
     state.Lock.EnterWriteLock();
     try
     {
         if (fieldsToRead.All(state.IsInCache))
             return;
         FillCache(state, fieldsToRead, reader);
     }
     finally
     {
         state.Lock.ExitWriteLock();
     }
 }
Beispiel #13
0
        private static void FillCache(IndexSearcherHolder.IndexSearcherHoldingState state, List<string> fieldsToRead,IndexReader reader)
        {
            foreach (var field in fieldsToRead)
            {
                using (var termDocs = reader.TermDocs())
                {
                    using (var termEnum = reader.Terms(new Term(field)))
                    {
                        do
                        {
                            if (termEnum.Term == null || field != termEnum.Term.Field)
                                break;

                            if (LowPrecisionNumber(termEnum.Term))
                                continue;


                            var totalDocCountIncludedDeletes = termEnum.DocFreq();
                            termDocs.Seek(termEnum.Term);

                            while (termDocs.Next() && totalDocCountIncludedDeletes > 0)
                            {
                                totalDocCountIncludedDeletes -= 1;
                                if (reader.IsDeleted(termDocs.Doc))
                                    continue;

                                state.SetInCache(field, termDocs.Doc, termEnum.Term);
                            }
                        } while (termEnum.Next());
                    }
                }
            }
        }