public override void Run()
{
Random random = Random();
while (numCats.DecrementAndGet() > 0)
{
try
{
int value = random.Next(range);
FacetLabel cp = new FacetLabel(Convert.ToString(value / 1000), Convert.ToString(value / 10000), Convert.ToString(value / 100000), Convert.ToString(value));
int ord = tw.AddCategory(cp);
Assert.True(tw.GetParent(ord) != -1, "invalid parent for ordinal " + ord + ", category " + cp);
string l1 = FacetsConfig.PathToString(cp.Components, 1);
string l2 = FacetsConfig.PathToString(cp.Components, 2);
string l3 = FacetsConfig.PathToString(cp.Components, 3);
string l4 = FacetsConfig.PathToString(cp.Components, 4);
values[l1] = l1;
values[l2] = l2;
values[l3] = l3;
values[l4] = l4;
}
catch (IOException e)
{
throw new Exception(e.Message, e);
}
}
}
public override void Run()
{
for (int i = 0; i < numCategories; i++)
{
try
{
destTW.AddCategory(new FacetLabel("a", Convert.ToString(i)));
}
catch (IOException e)
{
// shouldn't happen - if it does, let the test fail on uncaught exception.
throw new Exception(e.ToString(), e);
}
}
}
public virtual void TestWriterTwice3()
{
var indexDir = NewDirectory();
// First, create and fill the taxonomy
var tw = new DirectoryTaxonomyWriter(indexDir);
FillTaxonomy(tw);
tw.Dispose();
// Now, open the same taxonomy and add the same categories again.
// After a few categories, the LuceneTaxonomyWriter implementation
// will stop looking for each category on disk, and rather read them
// all into memory and close it's reader. The bug was that it closed
// the reader, but forgot that it did (because it didn't set the reader
// reference to null).
tw = new DirectoryTaxonomyWriter(indexDir);
FillTaxonomy(tw);
// Add one new category, just to make commit() do something:
tw.AddCategory(new FacetLabel("hi"));
// Do a commit(). Here was a bug - if tw had a reader open, it should
// be reopened after the commit. However, in our case the reader should
// not be open (as explained above) but because it was not set to null,
// we forgot that, tried to reopen it, and got an AlreadyClosedException.
tw.Commit();
Assert.AreEqual(ExpectedCategories.Length + 1, tw.Size);
tw.Dispose();
indexDir.Dispose();
}
public virtual void TestWriterSimpler()
{
var indexDir = NewDirectory();
var tw = new DirectoryTaxonomyWriter(indexDir);
Assert.AreEqual(1, tw.Size); // the root only
// Test that adding a new top-level category works
Assert.AreEqual(1, tw.AddCategory(new FacetLabel("a")));
Assert.AreEqual(2, tw.Size);
// Test that adding the same category again is noticed, and the
// same ordinal (and not a new one) is returned.
Assert.AreEqual(1, tw.AddCategory(new FacetLabel("a")));
Assert.AreEqual(2, tw.Size);
// Test that adding another top-level category returns a new ordinal,
// not the same one
Assert.AreEqual(2, tw.AddCategory(new FacetLabel("b")));
Assert.AreEqual(3, tw.Size);
// Test that adding a category inside one of the above adds just one
// new ordinal:
Assert.AreEqual(3, tw.AddCategory(new FacetLabel("a", "c")));
Assert.AreEqual(4, tw.Size);
// Test that adding the same second-level category doesn't do anything:
Assert.AreEqual(3, tw.AddCategory(new FacetLabel("a", "c")));
Assert.AreEqual(4, tw.Size);
// Test that adding a second-level category with two new components
// indeed adds two categories
Assert.AreEqual(5, tw.AddCategory(new FacetLabel("d", "e")));
Assert.AreEqual(6, tw.Size);
// Verify that the parents were added above in the order we expected
Assert.AreEqual(4, tw.AddCategory(new FacetLabel("d")));
// Similar, but inside a category that already exists:
Assert.AreEqual(7, tw.AddCategory(new FacetLabel("b", "d", "e")));
Assert.AreEqual(8, tw.Size);
// And now inside two levels of categories that already exist:
Assert.AreEqual(8, tw.AddCategory(new FacetLabel("b", "d", "f")));
Assert.AreEqual(9, tw.Size);
tw.Dispose();
indexDir.Dispose();
}
public virtual void TestWriterLock()
{
// native fslock impl gets angry if we use it, so use RAMDirectory explicitly.
var indexDir = new RAMDirectory();
var tw = new DirectoryTaxonomyWriter(indexDir);
tw.AddCategory(new FacetLabel("hi", "there"));
tw.Commit();
// we deliberately not close the write now, and keep it open and
// locked.
// Verify that the writer worked:
var tr = new DirectoryTaxonomyReader(indexDir);
Assert.AreEqual(2, tr.GetOrdinal(new FacetLabel("hi", "there")));
// Try to open a second writer, with the first one locking the directory.
// We expect to get a LockObtainFailedException.
try
{
Assert.Null(new DirectoryTaxonomyWriter(indexDir));
Fail("should have failed to write in locked directory");
}
catch (LockObtainFailedException)
{
// this is what we expect to happen.
}
// Remove the lock, and now the open should succeed, and we can
// write to the new writer.
DirectoryTaxonomyWriter.Unlock(indexDir);
var tw2 = new DirectoryTaxonomyWriter(indexDir);
tw2.AddCategory(new FacetLabel("hey"));
tw2.Dispose();
// See that the writer indeed wrote:
var newtr = TaxonomyReader.OpenIfChanged(tr);
Assert.NotNull(newtr);
tr.Dispose();
tr = newtr;
Assert.AreEqual(3, tr.GetOrdinal(new FacetLabel("hey")));
tr.Dispose();
tw.Dispose();
indexDir.Dispose();
}
public virtual void TestTaxonomyReaderRefreshRaces()
{
// compute base child arrays - after first chunk, and after the other
var indexDirBase = NewDirectory();
var twBase = new DirectoryTaxonomyWriter(indexDirBase);
twBase.AddCategory(new FacetLabel("a", "0"));
FacetLabel abPath = new FacetLabel("a", "b");
twBase.AddCategory(abPath);
twBase.Commit();
var trBase = new DirectoryTaxonomyReader(indexDirBase);
ParallelTaxonomyArrays ca1 = trBase.ParallelTaxonomyArrays;
int abOrd = trBase.GetOrdinal(abPath);
int abYoungChildBase1 = ca1.Children()[abOrd];
int numCategories = AtLeast(800);
for (int i = 0; i < numCategories; i++)
{
twBase.AddCategory(new FacetLabel("a", "b", Convert.ToString(i)));
}
twBase.Dispose();
var newTaxoReader = TaxonomyReader.OpenIfChanged(trBase);
Assert.NotNull(newTaxoReader);
trBase.Dispose();
trBase = newTaxoReader;
ParallelTaxonomyArrays ca2 = trBase.ParallelTaxonomyArrays;
int abYoungChildBase2 = ca2.Children()[abOrd];
int numRetries = AtLeast(50);
for (int retry = 0; retry < numRetries; retry++)
{
AssertConsistentYoungestChild(abPath, abOrd, abYoungChildBase1, abYoungChildBase2, retry, numCategories);
}
trBase.Dispose();
indexDirBase.Dispose();
}
public virtual void TestSeparateReaderAndWriter2()
{
var indexDir = NewDirectory();
var tw = new DirectoryTaxonomyWriter(indexDir);
tw.Commit();
var tr = new DirectoryTaxonomyReader(indexDir);
// Test getOrdinal():
FacetLabel author = new FacetLabel("Author");
Assert.AreEqual(1, tr.Size); // the empty taxonomy has size 1 (the root)
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(author));
tw.AddCategory(author);
// before commit and refresh, no change:
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(author));
Assert.AreEqual(1, tr.Size); // still root only...
Assert.Null(TaxonomyReader.OpenIfChanged(tr)); // this is not enough, because tw.Commit() hasn't been done yet
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(author));
Assert.AreEqual(1, tr.Size); // still root only...
tw.Commit();
// still not enough before refresh:
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(author));
Assert.AreEqual(1, tr.Size); // still root only...
var newTaxoReader = TaxonomyReader.OpenIfChanged(tr);
Assert.NotNull(newTaxoReader);
tr.Dispose();
tr = newTaxoReader;
Assert.AreEqual(1, tr.GetOrdinal(author));
Assert.AreEqual(2, tr.Size);
tw.Dispose();
tr.Dispose();
indexDir.Dispose();
}
public virtual void TestSeparateReaderAndWriter()
{
var indexDir = NewDirectory();
var tw = new DirectoryTaxonomyWriter(indexDir);
tw.Commit();
var tr = new DirectoryTaxonomyReader(indexDir);
Assert.AreEqual(1, tr.Size); // the empty taxonomy has size 1 (the root)
tw.AddCategory(new FacetLabel("Author"));
Assert.AreEqual(1, tr.Size); // still root only...
Assert.Null(TaxonomyReader.OpenIfChanged(tr)); // this is not enough, because tw.Commit() hasn't been done yet
Assert.AreEqual(1, tr.Size); // still root only...
tw.Commit();
Assert.AreEqual(1, tr.Size); // still root only...
var newTaxoReader = TaxonomyReader.OpenIfChanged(tr);
Assert.NotNull(newTaxoReader);
tr.Dispose();
tr = newTaxoReader;
int author = 1;
try
{
Assert.AreEqual(TaxonomyReader.ROOT_ORDINAL, tr.ParallelTaxonomyArrays.Parents()[author]);
// ok
}
catch (System.IndexOutOfRangeException)
{
Fail("After category addition, commit() and refresh(), getParent for " + author + " should NOT throw exception");
}
Assert.AreEqual(2, tr.Size); // finally, see there are two categories
// now, add another category, and verify that after commit and refresh
// the parent of this category is correct (this requires the reader
// to correctly update its prefetched parent vector), and that the
// old information also wasn't ruined:
tw.AddCategory(new FacetLabel("Author", "Richard Dawkins"));
int dawkins = 2;
tw.Commit();
newTaxoReader = TaxonomyReader.OpenIfChanged(tr);
Assert.NotNull(newTaxoReader);
tr.Dispose();
tr = newTaxoReader;
int[] parents = tr.ParallelTaxonomyArrays.Parents();
Assert.AreEqual(author, parents[dawkins]);
Assert.AreEqual(TaxonomyReader.ROOT_ORDINAL, parents[author]);
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, parents[TaxonomyReader.ROOT_ORDINAL]);
Assert.AreEqual(3, tr.Size);
tw.Dispose();
tr.Dispose();
indexDir.Dispose();
}
public virtual void TestNrt()
{
var dir = NewDirectory();
var writer = new DirectoryTaxonomyWriter(dir);
var reader = new DirectoryTaxonomyReader(writer);
FacetLabel cp = new FacetLabel("a");
writer.AddCategory(cp);
var newReader = TaxonomyReader.OpenIfChanged(reader);
Assert.NotNull(newReader, "expected a new instance");
Assert.AreEqual(2, newReader.Size);
Assert.AreNotSame(TaxonomyReader.INVALID_ORDINAL, newReader.GetOrdinal(cp));
reader.Dispose();
reader = newReader;
writer.Dispose();
reader.Dispose();
dir.Dispose();
}
public virtual void TestChildrenArraysGrowth()
{
var indexDir = NewDirectory();
var tw = new DirectoryTaxonomyWriter(indexDir);
tw.AddCategory(new FacetLabel("hi", "there"));
tw.Commit();
var tr = new DirectoryTaxonomyReader(indexDir);
ParallelTaxonomyArrays ca = tr.ParallelTaxonomyArrays;
Assert.AreEqual(3, tr.Size);
Assert.AreEqual(3, ca.Siblings().Length);
Assert.AreEqual(3, ca.Children().Length);
Assert.True(Arrays.Equals(new int[] { 1, 2, -1 }, ca.Children()));
Assert.True(Arrays.Equals(new int[] { -1, -1, -1 }, ca.Siblings()));
tw.AddCategory(new FacetLabel("hi", "ho"));
tw.AddCategory(new FacetLabel("hello"));
tw.Commit();
// Before refresh, nothing changed..
ParallelTaxonomyArrays newca = tr.ParallelTaxonomyArrays;
Assert.AreSame(newca, ca); // we got exactly the same object
Assert.AreEqual(3, tr.Size);
Assert.AreEqual(3, ca.Siblings().Length);
Assert.AreEqual(3, ca.Children().Length);
// After the refresh, things change:
var newtr = TaxonomyReader.OpenIfChanged(tr);
Assert.NotNull(newtr);
tr.Dispose();
tr = newtr;
ca = tr.ParallelTaxonomyArrays;
Assert.AreEqual(5, tr.Size);
Assert.AreEqual(5, ca.Siblings().Length);
Assert.AreEqual(5, ca.Children().Length);
Assert.True(Arrays.Equals(new int[] { 4, 3, -1, -1, -1 }, ca.Children()));
Assert.True(Arrays.Equals(new int[] { -1, -1, -1, 2, 1 }, ca.Siblings()));
tw.Dispose();
tr.Dispose();
indexDir.Dispose();
}
private void AssertConsistentYoungestChild(FacetLabel abPath, int abOrd, int abYoungChildBase1, int abYoungChildBase2, int retry, int numCategories)
{
var indexDir = new SlowRAMDirectory(-1, null); // no slowness for intialization
var tw = new DirectoryTaxonomyWriter(indexDir);
tw.AddCategory(new FacetLabel("a", "0"));
tw.AddCategory(abPath);
tw.Commit();
var tr = new DirectoryTaxonomyReader(indexDir);
for (int i = 0; i < numCategories; i++)
{
var cp = new FacetLabel("a", "b", Convert.ToString(i));
tw.AddCategory(cp);
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(cp), "Ordinal of " + cp + " must be invalid until Taxonomy Reader was refreshed");
}
tw.Dispose();
var stop = new AtomicBoolean(false);
Exception[] error = new Exception[] { null };
int[] retrieval = new int[] { 0 };
var thread = new ThreadAnonymousInnerClassHelper(this, abPath, abOrd, abYoungChildBase1, abYoungChildBase2, retry, tr, stop, error, retrieval);
thread.Start();
indexDir.SleepMillis = 1; // some delay for refresh
var newTaxoReader = TaxonomyReader.OpenIfChanged(tr);
if (newTaxoReader != null)
{
newTaxoReader.Dispose();
}
stop.Set(true);
thread.Join();
Assert.Null(error[0], "Unexpcted exception at retry " + retry + " retrieval " + retrieval[0] + ": \n" + stackTraceStr(error[0]));
tr.Dispose();
}
public virtual void TestReplaceTaxonomyDirectory()
{
Store.Directory indexDir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
var tw = new DirectoryTaxonomyWriter(taxoDir);
w.Commit();
tw.Commit();
Store.Directory taxoDir2 = NewDirectory();
var tw2 = new DirectoryTaxonomyWriter(taxoDir2);
tw2.AddCategory(new FacetLabel("a", "b"));
tw2.Dispose();
var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
Assert.AreEqual(1, pair.taxonomyReader.Size);
}
finally
{
mgr.Release(pair);
}
w.AddDocument(new Document());
tw.ReplaceTaxonomy(taxoDir2);
taxoDir2.Dispose();
w.Commit();
tw.Commit();
mgr.MaybeRefresh();
pair = mgr.Acquire();
try
{
Assert.AreEqual(3, pair.taxonomyReader.Size);
}
finally
{
mgr.Release(pair);
}
IOUtils.Close(mgr, tw, w, taxoDir, indexDir);
}