/// <summary>
/// Initializes a new instance of the <see cref="LuceneIndex" /> class.
/// </summary>
/// <param name="indexPath">The path to the directory that will contain the Lucene index files.</param>
/// <param name="schema">The schema.</param>
/// <exception cref="System.ArgumentNullException"></exception>
public LuceneIndex(string indexPath, Schema schema)
{
if (String.IsNullOrWhiteSpace(indexPath))
throw new ArgumentNullException(nameof(indexPath));
if (schema == null)
throw new ArgumentNullException(nameof(schema));
IndexPath = indexPath;
Schema = schema;
if (System.IO.Directory.Exists(IndexPath))
{
if (Schema.IsDefault())
throw new InvalidOperationException($"There is an existing index on '{IndexPath}'.");
}
else
{
System.IO.Directory.CreateDirectory(IndexPath);
}
_indexDirectory = new MMapDirectory(Paths.get(IndexPath));
var taxonomyIndexPath = System.IO.Path.Combine(IndexPath, "taxonomy");
if (!System.IO.Directory.Exists(taxonomyIndexPath))
System.IO.Directory.CreateDirectory(taxonomyIndexPath);
_taxonomyDirectory = new MMapDirectory(Paths.get(taxonomyIndexPath));
_compositeAnalyzer = new CompositeAnalyzer(Schema);
_ramBufferSizeMB = Double.Parse(ConfigurationManager.AppSettings["IndexWriter.RAMBufferSizeMB"] ?? "128");
var config = new IndexWriterConfig(_compositeAnalyzer)
.SetOpenMode(IndexWriterConfigOpenMode.CREATE_OR_APPEND)
.SetRAMBufferSizeMB(_ramBufferSizeMB)
.SetCommitOnClose(true);
_indexWriter = new IndexWriter(_indexDirectory, config);
_taxonomyWriter = new DirectoryTaxonomyWriter(_taxonomyDirectory, IndexWriterConfigOpenMode.CREATE_OR_APPEND);
_searcherTaxonomyManager = new SearcherTaxonomyManager(_indexWriter, true, null, _taxonomyWriter);
_facetBuilder = new LuceneFacetBuilder(_taxonomyWriter);
_refreshIntervalSeconds = Double.Parse(ConfigurationManager.AppSettings["IndexSearcher.RefreshIntervalSeconds"] ?? "0.5");
_commitIntervalSeconds = Double.Parse(ConfigurationManager.AppSettings["IndexWriter.CommitIntervalSeconds"] ?? "60");
_writeAllowedFlag = new ManualResetEventSlim(true);
_refreshTimer = new Timer(o => Refresh(), null, TimeSpan.FromSeconds(_refreshIntervalSeconds), TimeSpan.FromSeconds(_refreshIntervalSeconds));
_commitTimer = new Timer(o => Commit(), null, TimeSpan.FromSeconds(_commitIntervalSeconds), TimeSpan.FromSeconds(_commitIntervalSeconds));
}
public virtual void TestReplaceTaxonomyDirectory()
{
Store.Directory indexDir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
var tw = new DirectoryTaxonomyWriter(taxoDir);
w.Commit();
tw.Commit();
Store.Directory taxoDir2 = NewDirectory();
var tw2 = new DirectoryTaxonomyWriter(taxoDir2);
tw2.AddCategory(new FacetLabel("a", "b"));
tw2.Dispose();
var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
Assert.AreEqual(1, pair.taxonomyReader.Size);
}
finally
{
mgr.Release(pair);
}
w.AddDocument(new Document());
tw.ReplaceTaxonomy(taxoDir2);
taxoDir2.Dispose();
w.Commit();
tw.Commit();
mgr.MaybeRefresh();
pair = mgr.Acquire();
try
{
Assert.AreEqual(3, pair.taxonomyReader.Size);
}
finally
{
mgr.Release(pair);
}
IOUtils.Close(mgr, tw, w, taxoDir, indexDir);
}
public virtual void TestDirectory()
{
Store.Directory indexDir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
var tw = new DirectoryTaxonomyWriter(taxoDir);
// first empty commit
w.Commit();
tw.Commit();
var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("field", true);
AtomicBoolean stop = new AtomicBoolean();
// How many unique facets to index before stopping:
int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
var indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop);
indexer.Start();
try
{
while (!stop.Get())
{
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
//System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
FacetsCollector sfc = new FacetsCollector();
pair.searcher.Search(new MatchAllDocsQuery(), sfc);
Facets facets = GetTaxonomyFacetCounts(pair.taxonomyReader, config, sfc);
FacetResult result = facets.GetTopChildren(10, "field");
if (pair.searcher.IndexReader.NumDocs > 0)
{
//System.out.println(pair.taxonomyReader.getSize());
Assert.True(result.ChildCount > 0);
Assert.True(result.LabelValues.Length > 0);
}
//if (VERBOSE) {
//System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
//}
}
finally
{
mgr.Release(pair);
}
}
}
finally
{
indexer.Join();
}
if (VERBOSE)
{
Console.WriteLine("TEST: now stop");
}
IOUtils.Close(mgr, tw, w, taxoDir, indexDir);
}
public virtual void TestReplaceTaxonomyNrt()
{
Store.Directory dir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
var tw = new DirectoryTaxonomyWriter(taxoDir);
Store.Directory taxoDir2 = NewDirectory();
var tw2 = new DirectoryTaxonomyWriter(taxoDir2);
tw2.Dispose();
var mgr = new SearcherTaxonomyManager(w, true, null, tw);
w.AddDocument(new Document());
tw.ReplaceTaxonomy(taxoDir2);
taxoDir2.Dispose();
try
{
mgr.MaybeRefresh();
Fail("should have hit exception");
}
catch (InvalidOperationException)
{
// expected
}
IOUtils.Close(mgr, tw, w, taxoDir, dir);
}
public virtual void TestNrt()
{
Store.Directory dir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
// Don't allow tiny maxBufferedDocs; it can make this
// test too slow:
iwc.SetMaxBufferedDocs(Math.Max(500, iwc.MaxBufferedDocs));
// MockRandom/AlcololicMergePolicy are too slow:
TieredMergePolicy tmp = new TieredMergePolicy();
tmp.FloorSegmentMB = .001;
iwc.SetMergePolicy(tmp);
IndexWriter w = new IndexWriter(dir, iwc);
var tw = new DirectoryTaxonomyWriter(taxoDir);
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("field", true);
AtomicBoolean stop = new AtomicBoolean();
// How many unique facets to index before stopping:
int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
var indexer = new IndexerThread(w, config, tw, null, ordLimit, stop);
var mgr = new SearcherTaxonomyManager(w, true, null, tw);
var reopener = new ThreadAnonymousInnerClassHelper(this, stop, mgr);
reopener.Name = "reopener";
reopener.Start();
indexer.Name = "indexer";
indexer.Start();
try
{
while (!stop.Get())
{
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
//System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
FacetsCollector sfc = new FacetsCollector();
pair.searcher.Search(new MatchAllDocsQuery(), sfc);
Facets facets = GetTaxonomyFacetCounts(pair.taxonomyReader, config, sfc);
FacetResult result = facets.GetTopChildren(10, "field");
if (pair.searcher.IndexReader.NumDocs > 0)
{
//System.out.println(pair.taxonomyReader.getSize());
Assert.True(result.ChildCount > 0);
Assert.True(result.LabelValues.Length > 0);
}
//if (VERBOSE) {
//System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
//}
}
finally
{
mgr.Release(pair);
}
}
}
finally
{
indexer.Join();
reopener.Join();
}
if (VERBOSE)
{
Console.WriteLine("TEST: now stop");
}
IOUtils.Close(mgr, tw, w, taxoDir, dir);
}