public virtual void TestCountAndSumScore()
        {
            Store.Directory indexDir = NewDirectory();
            Store.Directory taxoDir  = NewDirectory();

            var          taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
            IndexWriter  iw         = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
            FacetsConfig config     = new FacetsConfig();

            config.SetIndexFieldName("b", "$b");

            for (int i = AtLeast(30); i > 0; --i)
            {
                Document doc = new Document();
                doc.Add(new StringField("f", "v", Field.Store.NO));
                doc.Add(new FacetField("a", "1"));
                doc.Add(new FacetField("b", "1"));
                iw.AddDocument(config.Build(taxoWriter, doc));
            }

            DirectoryReader r          = DirectoryReader.Open(iw, true);
            var             taxoReader = new DirectoryTaxonomyReader(taxoWriter);

            FacetsCollector fc = new FacetsCollector(true);

            FacetsCollector.Search(NewSearcher(r), new MatchAllDocsQuery(), 10, fc);

            Facets facets1 = GetTaxonomyFacetCounts(taxoReader, config, fc);
            Facets facets2 = new TaxonomyFacetSumValueSource(new DocValuesOrdinalsReader("$b"), taxoReader, config, fc, new TaxonomyFacetSumValueSource.ScoreValueSource());

            Assert.AreEqual(r.MaxDoc, (int)facets1.GetTopChildren(10, "a").Value);
            Assert.AreEqual(r.MaxDoc, (double)facets2.GetTopChildren(10, "b").Value, 1E-10);
            IOUtils.Close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
        }
        /// <summary>User runs a query and counts facets.</summary>
        private IList <FacetResult> Search()
        {
            using DirectoryReader indexReader = DirectoryReader.Open(indexDir);
            IndexSearcher searcher = new IndexSearcher(indexReader);
            SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(indexReader);

            // Aggregatses the facet counts
            FacetsCollector fc = new FacetsCollector();

            // MatchAllDocsQuery is for "browsing" (counts facets
            // for all non-deleted docs in the index); normally
            // you'd use a "normal" query:
            FacetsCollector.Search(searcher, new MatchAllDocsQuery(), 10, fc);

            // Retrieve results
            Facets facets = new SortedSetDocValuesFacetCounts(state, fc);

            IList <FacetResult> results = new List <FacetResult>
            {
                facets.GetTopChildren(10, "Author"),
                facets.GetTopChildren(10, "Publish Year")
            };

            return(results);
        }
        public virtual void TestWithScore()
        {
            Store.Directory indexDir = NewDirectory();
            Store.Directory taxoDir  = NewDirectory();

            DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
            IndexWriter             iw         = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));

            FacetsConfig config = new FacetsConfig();

            for (int i = 0; i < 4; i++)
            {
                Document doc = new Document();
                doc.Add(new NumericDocValuesField("price", (i + 1)));
                doc.Add(new FacetField("a", Convert.ToString(i % 2)));
                iw.AddDocument(config.Build(taxoWriter, doc));
            }

            DirectoryReader         r          = DirectoryReader.Open(iw, true);
            DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

            ValueSource valueSource = new ValueSourceAnonymousInnerClassHelper(this);

            FacetsCollector fc = new FacetsCollector(true);
            // score documents by their 'price' field - makes asserting the correct counts for the categories easier
            Query q = new FunctionQuery(new LongFieldSource("price"));

            FacetsCollector.Search(NewSearcher(r), q, 10, fc);
            Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, fc, valueSource);

            Assert.AreEqual("dim=a path=[] value=10.0 childCount=2\n  1 (6.0)\n  0 (4.0)\n", facets.GetTopChildren(10, "a").ToString());

            IOUtils.Close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
        }
示例#4
0
        /// <summary>User runs a query and counts facets.</summary>
        private IList <FacetResult> FacetsWithSearch()
        {
            using (DirectoryReader indexReader = DirectoryReader.Open(indexDir))
                using (TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir))
                {
                    IndexSearcher searcher = new IndexSearcher(indexReader);

                    FacetsCollector fc = new FacetsCollector();

                    // MatchAllDocsQuery is for "browsing" (counts facets
                    // for all non-deleted docs in the index); normally
                    // you'd use a "normal" query:
                    FacetsCollector.Search(searcher, new MatchAllDocsQuery(), 10, fc);

                    // Retrieve results
                    IList <FacetResult> results = new List <FacetResult>();

                    // Count both "Publish Date" and "Author" dimensions
                    Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
                    results.Add(facets.GetTopChildren(10, "Author"));
                    results.Add(facets.GetTopChildren(10, "Publish Date"));

                    return(results);
                }// Disposes indexReader and taxoReader
        }
示例#5
0
        /// <summary>User runs a query and aggregates facets by summing their association values.</summary>
        private IList <FacetResult> SumAssociations()
        {
            using DirectoryReader indexReader = DirectoryReader.Open(indexDir);
            using TaxonomyReader taxoReader   = new DirectoryTaxonomyReader(taxoDir);
            IndexSearcher searcher = new IndexSearcher(indexReader);

            FacetsCollector fc = new FacetsCollector();

            // MatchAllDocsQuery is for "browsing" (counts facets
            // for all non-deleted docs in the index); normally
            // you'd use a "normal" query:
            FacetsCollector.Search(searcher, new MatchAllDocsQuery(), 10, fc);

            Facets tags  = new TaxonomyFacetSumInt32Associations("$tags", taxoReader, config, fc);
            Facets genre = new TaxonomyFacetSumSingleAssociations("$genre", taxoReader, config, fc);

            // Retrieve results
            IList <FacetResult> results = new List <FacetResult>
            {
                tags.GetTopChildren(10, "tags"),
                genre.GetTopChildren(10, "genre")
            };

            return(results);
        }
示例#6
0
        /// <summary>User runs a query and aggregates facets.</summary>
        private FacetResult Search()
        {
            using DirectoryReader indexReader = DirectoryReader.Open(indexDir);
            using TaxonomyReader taxoReader   = new DirectoryTaxonomyReader(taxoDir);
            IndexSearcher searcher = new IndexSearcher(indexReader);

            // Aggregate categories by an expression that combines the document's score
            // and its popularity field
            Expression     expr     = JavascriptCompiler.Compile("_score * sqrt(popularity)");
            SimpleBindings bindings = new SimpleBindings
            {
                new SortField("_score", SortFieldType.SCORE),     // the score of the document
                new SortField("popularity", SortFieldType.INT64), // the value of the 'popularity' field
            };

            // Aggregates the facet values
            FacetsCollector fc = new FacetsCollector(true);

            // MatchAllDocsQuery is for "browsing" (counts facets
            // for all non-deleted docs in the index); normally
            // you'd use a "normal" query:
            FacetsCollector.Search(searcher, new MatchAllDocsQuery(), 10, fc);

            // Retrieve results
            Facets      facets = new TaxonomyFacetSumValueSource(taxoReader, config, fc, expr.GetValueSource(bindings));
            FacetResult result = facets.GetTopChildren(10, "A");

            return(result);
        }
示例#7
0
        public static void TestFlexLuceneRAM(string[] args)
        {
            StandardAnalyzer analyzer = new StandardAnalyzer();

            FlexLucene.Store.Directory index = (FlexLucene.Store.Directory) new RAMDirectory();
            config = new IndexWriterConfig((Analyzer)analyzer);
            cnf    = new FacetsConfig();
            cnf.SetIndexFieldName("title", "facet_title");
            cnf.SetIndexFieldName("isbn", "facet_isbn");
            LuceneTest.taxoDir    = (FlexLucene.Store.Directory) new RAMDirectory();
            LuceneTest.taxoWriter = (TaxonomyWriter) new FlexLucene.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter(LuceneTest.taxoDir, IndexWriterConfigOpenMode.CREATE);

            IndexWriter w = new IndexWriter(index, LuceneTest.config);

            addDoc(w, "Lucene in Action", "9900001");
            addDoc(w, "Lucene for Dummies", "9900002");
            addDoc(w, "Lucene for Dummies 2", "9900003");

            w.close();
            String               querystr    = "isbn:99*";
            Query                q           = new QueryParser("title", (Analyzer)analyzer).Parse(querystr);
            int                  hitsPerPage = 10;
            IndexReader          reader      = (IndexReader)DirectoryReader.Open(index);
            IndexSearcher        searcher    = new IndexSearcher(reader);
            TopScoreDocCollector collector   = TopScoreDocCollector.Create(hitsPerPage);

            searcher.Search(q, (Collector)collector);
            ScoreDoc[] hits = collector.TopDocs().ScoreDocs;
            Console.WriteLine("Found " + hits.Length + " hits.");
            for (int i = 0; i < hits.Length; ++i)
            {
                int      docId = hits [i].Doc;
                Document d     = searcher.Doc(docId);
                Console.WriteLine(i + 1 + ". " + d.Get("isbn") + "\t" + d.Get("title"));
            }
            SortedSetDocValuesReaderState state = (SortedSetDocValuesReaderState) new DefaultSortedSetDocValuesReaderState(reader, "facet_isbn");
            FacetsCollector fc = new FacetsCollector();

            FacetsCollector.Search(searcher, q, 10, (Collector)fc);
            Facets      facets = (Facets) new SortedSetDocValuesFacetCounts(state, fc);
            FacetResult result = facets.GetTopChildren(10, "isbn", new String[0]);

            for (int j = 0; j < result.ChildCount; ++j)
            {
                LabelAndValue lv = result.LabelValues [j];
                Console.WriteLine(String.Format("Label={0}, Value={1}", lv.Label, lv.Value));
            }
            reader.close();
        }
示例#8
0
        /// <summary>User runs a query and counts facets.</summary>
        public FacetResult Search()
        {
            // Aggregates the facet counts
            FacetsCollector fc = new FacetsCollector();

            // MatchAllDocsQuery is for "browsing" (counts facets
            // for all non-deleted docs in the index); normally
            // you'd use a "normal" query:
            FacetsCollector.Search(searcher, new MatchAllDocsQuery(), 10, fc);

            Facets facets = new Int64RangeFacetCounts("timestamp", fc,
                                                      PAST_HOUR,
                                                      PAST_SIX_HOURS,
                                                      PAST_DAY);

            return(facets.GetTopChildren(10, "timestamp"));
        }
        /// <summary>User drills down on 'Publish Year/2010'.</summary>
        private FacetResult DrillDown()
        {
            using DirectoryReader indexReader = DirectoryReader.Open(indexDir);
            IndexSearcher searcher = new IndexSearcher(indexReader);
            SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(indexReader);

            // Now user drills down on Publish Year/2010:
            DrillDownQuery q = new DrillDownQuery(config);

            q.Add("Publish Year", "2010");
            FacetsCollector fc = new FacetsCollector();

            FacetsCollector.Search(searcher, q, 10, fc);

            // Retrieve results
            Facets      facets = new SortedSetDocValuesFacetCounts(state, fc);
            FacetResult result = facets.GetTopChildren(10, "Author");

            return(result);
        }
        public virtual void TestSumScoreAggregator()
        {
            Store.Directory indexDir = NewDirectory();
            Store.Directory taxoDir  = NewDirectory();

            DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
            IndexWriter             iw         = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));

            FacetsConfig config = new FacetsConfig();

            for (int i = AtLeast(30); i > 0; --i)
            {
                Document doc = new Document();
                if (Random().NextBoolean()) // don't match all documents
                {
                    doc.Add(new StringField("f", "v", Field.Store.NO));
                }
                doc.Add(new FacetField("dim", "a"));
                iw.AddDocument(config.Build(taxoWriter, doc));
            }

            DirectoryReader         r          = DirectoryReader.Open(iw, true);
            DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

            FacetsCollector    fc  = new FacetsCollector(true);
            ConstantScoreQuery csq = new ConstantScoreQuery(new MatchAllDocsQuery());

            csq.Boost = 2.0f;

            TopDocs td = FacetsCollector.Search(NewSearcher(r), csq, 10, fc);

            Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, fc, new TaxonomyFacetSumValueSource.ScoreValueSource());

            int expected = (int)(td.MaxScore * td.TotalHits);

            Assert.AreEqual(expected, (int)facets.GetSpecificValue("dim", "a"));

            IOUtils.Close(iw, taxoWriter, taxoReader, taxoDir, r, indexDir);
        }
示例#11
0
        /// <summary>User drills down on 'tags/solr'.</summary>
        private FacetResult DrillDown()
        {
            using DirectoryReader indexReader = DirectoryReader.Open(indexDir);
            using TaxonomyReader taxoReader   = new DirectoryTaxonomyReader(taxoDir);
            IndexSearcher searcher = new IndexSearcher(indexReader);

            // Passing no baseQuery means we drill down on all
            // documents ("browse only"):
            DrillDownQuery q = new DrillDownQuery(config);

            // Now user drills down on Publish Date/2010:
            q.Add("tags", "solr");
            FacetsCollector fc = new FacetsCollector();

            FacetsCollector.Search(searcher, q, 10, fc);

            // Retrieve results
            Facets      facets = new TaxonomyFacetSumSingleAssociations("$genre", taxoReader, config, fc);
            FacetResult result = facets.GetTopChildren(10, "genre");

            return(result);
        }
示例#12
0
        /// <summary>
        /// User drills down on 'Publish Date/2010', and we
        /// return facets for 'Author'
        /// </summary>
        private FacetResult DrillDown()
        {
            using (DirectoryReader indexReader = DirectoryReader.Open(indexDir))
                using (TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir))
                {
                    IndexSearcher searcher = new IndexSearcher(indexReader);

                    // Passing no baseQuery means we drill down on all
                    // documents ("browse only"):
                    DrillDownQuery q = new DrillDownQuery(config);

                    // Now user drills down on Publish Date/2010:
                    q.Add("Publish Date", "2010");
                    FacetsCollector fc = new FacetsCollector();
                    FacetsCollector.Search(searcher, q, 10, fc);

                    // Retrieve results
                    Facets      facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
                    FacetResult result = facets.GetTopChildren(10, "Author");

                    return(result);
                }// Disposes indexReader and taxoReader
        }
示例#13
0
        public virtual void TestRandom()
        {
            string[]        tokens   = GetRandomTokens(10);
            Store.Directory indexDir = NewDirectory();
            Store.Directory taxoDir  = NewDirectory();

            RandomIndexWriter w      = new RandomIndexWriter(Random(), indexDir, Similarity, TimeZone);
            var             tw       = new DirectoryTaxonomyWriter(taxoDir);
            FacetsConfig    config   = new FacetsConfig();
            int             numDocs  = AtLeast(1000);
            int             numDims  = TestUtil.NextInt(Random(), 1, 7);
            IList <TestDoc> testDocs = GetRandomDocs(tokens, numDocs, numDims);

            foreach (TestDoc testDoc in testDocs)
            {
                Document doc = new Document();
                doc.Add(NewStringField("content", testDoc.content, Field.Store.NO));
                for (int j = 0; j < numDims; j++)
                {
                    if (testDoc.dims[j] != null)
                    {
                        doc.Add(new FacetField("dim" + j, testDoc.dims[j]));
                    }
                }
                w.AddDocument(config.Build(tw, doc));
            }

            // NRT open
            IndexSearcher searcher = NewSearcher(w.Reader);

            // NRT open
            var tr = new DirectoryTaxonomyReader(tw);

            int iters = AtLeast(100);

            for (int iter = 0; iter < iters; iter++)
            {
                string searchToken = tokens[Random().Next(tokens.Length)];
                if (VERBOSE)
                {
                    Console.WriteLine("\nTEST: iter content=" + searchToken);
                }
                FacetsCollector fc = new FacetsCollector();
                FacetsCollector.Search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
                Facets facets = GetTaxonomyFacetCounts(tr, config, fc);

                // Slow, yet hopefully bug-free, faceting:
                var expectedCounts = new List <Dictionary <string, int?> >();
                for (int i = 0; i < numDims; i++)
                {
                    expectedCounts.Add(new Dictionary <string, int?>());
                }

                foreach (TestDoc doc in testDocs)
                {
                    if (doc.content.Equals(searchToken))
                    {
                        for (int j = 0; j < numDims; j++)
                        {
                            if (doc.dims[j] != null)
                            {
                                int?v = expectedCounts[j].ContainsKey(doc.dims[j]) ? expectedCounts[j][doc.dims[j]] : null;
                                if (v == null)
                                {
                                    expectedCounts[j][doc.dims[j]] = 1;
                                }
                                else
                                {
                                    expectedCounts[j][doc.dims[j]] = (int)v + 1;
                                }
                            }
                        }
                    }
                }

                List <FacetResult> expected = new List <FacetResult>();
                for (int i = 0; i < numDims; i++)
                {
                    List <LabelAndValue> labelValues = new List <LabelAndValue>();
                    int totCount = 0;
                    foreach (KeyValuePair <string, int?> ent in expectedCounts[i])
                    {
                        labelValues.Add(new LabelAndValue(ent.Key, ent.Value.Value));
                        totCount += ent.Value.Value;
                    }
                    SortLabelValues(labelValues);
                    if (totCount > 0)
                    {
                        expected.Add(new FacetResult("dim" + i, new string[0], totCount, labelValues.ToArray(), labelValues.Count));
                    }
                }

                // Sort by highest value, tie break by value:
                SortFacetResults(expected);

                IList <FacetResult> actual = facets.GetAllDims(10);

                // Messy: fixup ties
                SortTies(actual);

                Assert.AreEqual(expected, actual);
            }

            IOUtils.Close(w, tw, searcher.IndexReader, tr, indexDir, taxoDir);
        }
示例#14
0
        public async Task <SearchResult <T> > SearchAsync(SearchQuery queryDefinition, CancellationToken cancellationToken = default)
        {
            using (await writerLock.ReaderLockAsync(cancellationToken))
            {
                var      result = new SearchResult <T>();
                List <T> hits   = new List <T>();

                using (var writer = getWriter())
                {
                    Query query = new MatchAllDocsQuery();

                    // Term queries
                    if (queryDefinition.TermQueries.Any())
                    {
                        var phraseQuery = new MultiPhraseQuery();
                        foreach (var termQuery in queryDefinition.TermQueries)
                        {
                            phraseQuery.Add(
                                termQuery.value
                                .Split(" ".ToCharArray(), StringSplitOptions.RemoveEmptyEntries)
                                .Select(phrase => new Term(termQuery.field, phrase.ToLower()))
                                .ToArray()
                                );
                        }
                        query = phraseQuery;
                    }

                    var reader       = writer.DocsWriter.GetReader(applyAllDeletes: true);
                    var searcher     = new IndexSearcher(reader);
                    var luceneResult = searcher.Search(query, queryDefinition.Limit);

                    foreach (var doc in luceneResult.ScoreDocs)
                    {
                        var foundDoc = searcher.Doc(doc.Doc);
                        hits.Add(await inflateDocument(foundDoc));
                    }

                    result.TotalHits = luceneResult.TotalHits;
                    result.Hits      = hits;

                    // Facets
                    if (queryDefinition.Facets.Any())
                    {
                        FacetsConfig    facetsConfig = new FacetsConfig();
                        FacetsCollector fc           = new FacetsCollector();
                        FacetsCollector.Search(searcher, query, queryDefinition.FacetMax, fc);
                        using (var taxonomyReader = new DirectoryTaxonomyReader(FSDirectory.Open(Path.Combine(options.IndexPath, indexType, "taxonomy"))))
                        {
                            var facets = new FastTaxonomyFacetCounts(taxonomyReader, facetsConfig, fc);
                            foreach (var facet in queryDefinition.Facets)
                            {
                                var facetGroup = new FacetGroup {
                                    Field = facet
                                };
                                facetGroup.Facets =
                                    facets.GetTopChildren(queryDefinition.FacetMax, facet).LabelValues
                                    .Select(x => new Facet {
                                    Key = x.Label, Count = (long)x.Value
                                })
                                    .ToArray();
                                result.FacetGroups.Add(facetGroup);
                            }
                        }
                    }
                }

                return(result);
            }
        }
示例#15
0
        public virtual void TestRandom()
        {
            AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet);
            string[]  tokens   = GetRandomTokens(10);
            Directory indexDir = NewDirectory();
            Directory taxoDir  = NewDirectory();

            RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
                this,
#endif
                Random, indexDir);
            FacetsConfig    config   = new FacetsConfig();
            int             numDocs  = AtLeast(1000);
            int             numDims  = TestUtil.NextInt32(Random, 1, 7);
            IList <TestDoc> testDocs = GetRandomDocs(tokens, numDocs, numDims);

            foreach (TestDoc testDoc in testDocs)
            {
                Document doc = new Document();
                doc.Add(NewStringField("content", testDoc.content, Field.Store.NO));
                for (int j = 0; j < numDims; j++)
                {
                    if (testDoc.dims[j] != null)
                    {
                        doc.Add(new SortedSetDocValuesFacetField("dim" + j, testDoc.dims[j]));
                    }
                }
                w.AddDocument(config.Build(doc));
            }

            // NRT open
            IndexSearcher searcher = NewSearcher(w.GetReader());

            // Per-top-reader state:
            SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);

            int iters = AtLeast(100);

            for (int iter = 0; iter < iters; iter++)
            {
                string searchToken = tokens[Random.Next(tokens.Length)];
                if (Verbose)
                {
                    Console.WriteLine("\nTEST: iter content=" + searchToken);
                }
                FacetsCollector fc = new FacetsCollector();
                FacetsCollector.Search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
                Facets facets = new SortedSetDocValuesFacetCounts(state, fc);

                // Slow, yet hopefully bug-free, faceting:
                var expectedCounts = new List <Dictionary <string, int?> >();
                for (int i = 0; i < numDims; i++)
                {
                    expectedCounts.Add(new Dictionary <string, int?>());
                }

                foreach (TestDoc doc in testDocs)
                {
                    if (doc.content.Equals(searchToken, StringComparison.Ordinal))
                    {
                        for (int j = 0; j < numDims; j++)
                        {
                            if (doc.dims[j] != null)
                            {
                                if (!expectedCounts[j].TryGetValue(doc.dims[j], out int?v))
                                {
                                    expectedCounts[j][doc.dims[j]] = 1;
                                }
                                else
                                {
                                    expectedCounts[j][doc.dims[j]] = (int)v + 1;
                                }
                            }
                        }
                    }
                }

                List <FacetResult> expected = new List <FacetResult>();
                for (int i = 0; i < numDims; i++)
                {
                    List <LabelAndValue> labelValues = new List <LabelAndValue>();
                    int totCount = 0;
                    foreach (KeyValuePair <string, int?> ent in expectedCounts[i])
                    {
                        labelValues.Add(new LabelAndValue(ent.Key, ent.Value.Value));
                        totCount += ent.Value.Value;
                    }
                    SortLabelValues(labelValues);
                    if (totCount > 0)
                    {
                        expected.Add(new FacetResult("dim" + i, new string[0], totCount, labelValues.ToArray(), labelValues.Count));
                    }
                }

                // Sort by highest value, tie break by value:
                SortFacetResults(expected);

                IList <FacetResult> actual = facets.GetAllDims(10);

                // Messy: fixup ties
                //sortTies(actual);

                CollectionAssert.AreEqual(expected, actual);
            }

            IOUtils.Dispose(w, searcher.IndexReader, indexDir, taxoDir);
        }
示例#16
0
        public virtual void TestRandom()
        {
            string[]        tokens   = GetRandomTokens(10);
            Store.Directory indexDir = NewDirectory();
            Store.Directory taxoDir  = NewDirectory();

            RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
                this,
#endif
                Random, indexDir);
            var             tw       = new DirectoryTaxonomyWriter(taxoDir);
            FacetsConfig    config   = new FacetsConfig();
            int             numDocs  = AtLeast(1000);
            int             numDims  = TestUtil.NextInt32(Random, 1, 7);
            IList <TestDoc> testDocs = GetRandomDocs(tokens, numDocs, numDims);

            foreach (TestDoc testDoc in testDocs)
            {
                Document doc = new Document();
                doc.Add(NewStringField("content", testDoc.content, Field.Store.NO));
                testDoc.value = Random.NextSingle();
                doc.Add(new SingleDocValuesField("value", testDoc.value));
                for (int j = 0; j < numDims; j++)
                {
                    if (testDoc.dims[j] != null)
                    {
                        doc.Add(new FacetField("dim" + j, testDoc.dims[j]));
                    }
                }
                w.AddDocument(config.Build(tw, doc));
            }

            // NRT open
            IndexSearcher searcher = NewSearcher(w.GetReader());

            // NRT open
            var tr = new DirectoryTaxonomyReader(tw);

            ValueSource values = new SingleFieldSource("value");

            int iters = AtLeast(100);

            for (int iter = 0; iter < iters; iter++)
            {
                string searchToken = tokens[Random.Next(tokens.Length)];
                if (Verbose)
                {
                    Console.WriteLine("\nTEST: iter content=" + searchToken);
                }
                FacetsCollector fc = new FacetsCollector();
                FacetsCollector.Search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
                Facets facets = new TaxonomyFacetSumValueSource(tr, config, fc, values);

                // Slow, yet hopefully bug-free, faceting:
                var expectedValues = new List <Dictionary <string, float?> >(numDims);
                for (int i = 0; i < numDims; i++)
                {
                    expectedValues.Add(new Dictionary <string, float?>());
                }

                foreach (TestDoc doc in testDocs)
                {
                    if (doc.content.Equals(searchToken, StringComparison.Ordinal))
                    {
                        for (int j = 0; j < numDims; j++)
                        {
                            if (doc.dims[j] != null)
                            {
                                if (!expectedValues[j].TryGetValue(doc.dims[j], out float?v) || v == null)
                                {
                                    expectedValues[j][doc.dims[j]] = doc.value;
                                }
                                else
                                {
                                    expectedValues[j][doc.dims[j]] = (float)v + doc.value;
                                }
                            }
                        }
                    }
                }

                List <FacetResult> expected = new List <FacetResult>();
                for (int i = 0; i < numDims; i++)
                {
                    List <LabelAndValue> labelValues = new List <LabelAndValue>();
                    float totValue = 0;
                    foreach (KeyValuePair <string, float?> ent in expectedValues[i])
                    {
                        labelValues.Add(new LabelAndValue(ent.Key, ent.Value.Value));
                        totValue += ent.Value.Value;
                    }
                    SortLabelValues(labelValues);
                    if (totValue > 0)
                    {
                        expected.Add(new FacetResult("dim" + i, new string[0], totValue, labelValues.ToArray(), labelValues.Count));
                    }
                }

                // Sort by highest value, tie break by value:
                SortFacetResults(expected);

                IList <FacetResult> actual = facets.GetAllDims(10);

                // Messy: fixup ties
                SortTies(actual);

                if (Verbose)
                {
                    Console.WriteLine("expected=\n" + expected.ToString());
                    Console.WriteLine("actual=\n" + actual.ToString());
                }

                AssertFloatValuesEquals(expected, actual);
            }

            IOUtils.Dispose(w, tw, searcher.IndexReader, tr, indexDir, taxoDir);
        }
示例#17
0
        /// <summary>
        /// Search facet content in the existing index.
        /// </summary>
        /// <param name="text">The text to search for.</param>
        /// <param name="indexFields">The array of index fields to search in.</param>
        /// <param name="facetPaths">The array of facet paths to perform a drill down search on.</param>
        /// <param name="numberToReturn">The maximum number of documents to return.</param>
        /// <returns>The facet document.</returns>
        /// <remarks>Use wildcard chars ('*', '?', '\'), logical ('AND', 'OR'), Quoted exact phrase ("search this").</remarks>
        public Nequeo.Search.Engine.FacetDocument SearchFacetDocument(string text, FacetData.IndexField[] indexFields, FacetPath[] facetPaths, int numberToReturn = Int32.MaxValue)
        {
            Nequeo.Search.Engine.FacetDocument documents = new FacetDocument();
            documents.TotalHits = 0;

            try
            {
                // If text exists.
                if (!String.IsNullOrEmpty(text))
                {
                    // Load the searcher.
                    Lucene.Net.Search.IndexSearcher searcher = new Lucene.Net.Search.IndexSearcher(_reader);
                    string         searchFieldName           = "facetcontent";
                    Query          query      = null;
                    DrillDownQuery queryFacet = null;
                    TopDocs        results    = null;

                    // Build the facet configuration information.
                    FacetsConfig config = new FacetsConfig();

                    // Add the config.
                    foreach (FacetData.IndexField item in indexFields)
                    {
                        config.SetIndexFieldName(item.DimensionName, item.IndexFieldName);
                    }

                    // Get bytes
                    char[] textArray = text.ToCharArray();

                    // Search logical.
                    if (text.Contains("AND") || text.Contains("OR"))
                    {
                        // Create the query.
                        query = CreateLogicalQuery(text, searchFieldName);
                    }
                    else if (textArray[0].Equals('"') && textArray[textArray.Length - 1].Equals('"'))
                    {
                        // Create the query.
                        query = CreateQuotedQuery(new string(textArray, 1, textArray.Length - 2), searchFieldName);
                    }
                    else
                    {
                        // Create the query.
                        query = CreateBoolenQuery(text, BooleanClause.Occur.SHOULD, searchFieldName);
                    }

                    // Create the facet query.
                    queryFacet = new DrillDownQuery(config, query);
                    foreach (FacetPath facetPath in facetPaths)
                    {
                        // Add the path.
                        queryFacet.Add(facetPath.DimensionName, facetPath.Path);
                    }

                    // The collector.
                    FacetsCollector collector = new FacetsCollector();

                    // Search.
                    if (queryFacet != null)
                    {
                        results = FacetsCollector.Search(searcher, queryFacet, numberToReturn, collector);
                    }

                    // Get the total number of results that was asked for.
                    int totalResult = ((results.ScoreDocs != null && results.ScoreDocs.Length > 0) ? results.ScoreDocs.Length : 0);

                    // If result found.
                    if (results != null && results.TotalHits > 0)
                    {
                        List <TextDataResult>     textDataResults = new List <TextDataResult>();
                        List <FileDocumentResult> fileDocResults  = new List <FileDocumentResult>();

                        List <FacetPathResult>       facetPathResults = new List <FacetPathResult>();
                        IDictionary <string, Facets> facetsMap        = new Dictionary <string, Facets>();

                        // Add the facet count.
                        foreach (FacetData.IndexField item in indexFields)
                        {
                            // Add the facet for each index field.
                            facetsMap[item.DimensionName] = GetTaxonomyFacetCounts(_facetReader, config, collector, item.IndexFieldName);
                        }

                        // Create the multi facet list.
                        foreach (FacetPath facetPath in facetPaths)
                        {
                            try
                            {
                                // Add the facets.
                                Facets facets = facetsMap.First(u => u.Key.ToLower().Contains(facetPath.DimensionName.ToLower())).Value;
                                float  number = facets.GetSpecificValue(facetPath.DimensionName, facetPath.Path);

                                // Add the path.
                                facetPathResults.Add(new FacetPathResult(facetPath.DimensionName, number, facetPath.Path));
                            }
                            catch { }
                        }

                        // For each document found.
                        for (int i = 0; i < totalResult; i++)
                        {
                            FileDocumentResult fileDocument = null;
                            TextDataResult     textData     = null;

                            int docID = results.ScoreDocs[i].Doc;
                            Lucene.Net.Documents.Document doc = searcher.Doc(docID);

                            try
                            {
                                // Get the data for each field.
                                IndexableField[] textNameFields = doc.GetFields("textname");

                                // If this field exists then text data.
                                if (textNameFields.Length > 0)
                                {
                                    // Assign the data to the text document.
                                    textData       = new TextDataResult();
                                    textData.Name  = textNameFields.Length > 0 ? textNameFields[0].StringValue : null;
                                    textData.Score = results.ScoreDocs[i].Score;
                                    textData.Doc   = docID;

                                    // Do not know if the text was stored.
                                    IndexableField[] textValueFields = doc.GetFields("textcomplete");
                                    textData.Text = textValueFields.Length > 0 ? textValueFields[0].StringValue : null;
                                }
                            }
                            catch { }

                            // If text data exists then add.
                            if (textData != null)
                            {
                                textDataResults.Add(textData);
                            }

                            try
                            {
                                // Get the data for each field.
                                IndexableField[] pathNameFields     = doc.GetFields("path");
                                IndexableField[] modifiedNameFields = doc.GetFields("modified");

                                // If this field exists then file document.
                                if (pathNameFields.Length > 0)
                                {
                                    // Assign the data to the path document.
                                    fileDocument          = new FileDocumentResult();
                                    fileDocument.Path     = pathNameFields.Length > 0 ? pathNameFields[0].StringValue : null;
                                    fileDocument.Modified = modifiedNameFields.Length > 0 ? modifiedNameFields[0].StringValue : null;
                                    fileDocument.Score    = results.ScoreDocs[i].Score;
                                    fileDocument.Doc      = docID;
                                }
                            }
                            catch { }

                            // If file data exists then add.
                            if (fileDocument != null)
                            {
                                fileDocResults.Add(fileDocument);
                            }
                        }

                        // Assign the facet document values.
                        documents.MaxScore            = results.MaxScore;
                        documents.TotalHits           = results.TotalHits;
                        documents.FacetPathResults    = facetPathResults.ToArray();
                        documents.TextDataResults     = textDataResults.ToArray();
                        documents.FileDocumentResults = fileDocResults.ToArray();
                    }
                }

                // Return the documents.
                return(documents);
            }
            catch (Exception)
            {
                throw;
            }
        }