| public AddCategory ( FacetLabel categoryPath ) : int | ||
| categoryPath | FacetLabel | |
| return | int | |
private void TouchTaxo(DirectoryTaxonomyWriter taxoWriter, FacetLabel cp)
{
taxoWriter.AddCategory(cp);
taxoWriter.SetCommitData(new Dictionary <string, string>()
{
{ "just", "data" }
});
taxoWriter.Commit();
}
public virtual void TestOpenIfChangedReuse()
{
// test the reuse of data from the old DTR instance
foreach (bool nrt in new bool[] { false, true })
{
Directory dir = NewDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
FacetLabel cp_a = new FacetLabel("a");
writer.AddCategory(cp_a);
if (!nrt)
{
writer.Commit();
}
DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
// fill r1's caches
Assert.AreEqual(1, r1.GetOrdinal(cp_a));
Assert.AreEqual(cp_a, r1.GetPath(1));
FacetLabel cp_b = new FacetLabel("b");
writer.AddCategory(cp_b);
if (!nrt)
{
writer.Commit();
}
DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
Assert.IsNotNull(r2);
// add r2's categories to the caches
Assert.AreEqual(2, r2.GetOrdinal(cp_b));
Assert.AreEqual(cp_b, r2.GetPath(2));
// check that r1 doesn't see cp_b
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
Assert.IsNull(r1.GetPath(2));
r1.Dispose();
r2.Dispose();
writer.Dispose();
dir.Dispose();
}
}
public virtual void TestCommitUserData()
{
// Verifies taxonomy commit data
Directory dir = NewDirectory();
var taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.AddCategory(new FacetLabel("b"));
IDictionary <string, string> userCommitData = new Dictionary <string, string>();
userCommitData["testing"] = "1 2 3";
taxoWriter.SetCommitData(userCommitData);
taxoWriter.Dispose();
var r = DirectoryReader.Open(dir);
Assert.AreEqual(3, r.NumDocs, "2 categories plus root should have been committed to the underlying directory");
var readUserCommitData = r.IndexCommit.UserData;
Assert.IsTrue("1 2 3".Equals(readUserCommitData["testing"], StringComparison.Ordinal), "wrong value extracted from commit data");
Assert.IsNotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in commitData", readUserCommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]);
r.Dispose();
// open DirTaxoWriter again and commit, INDEX_EPOCH should still exist
// in the commit data, otherwise DirTaxoReader.refresh() might not detect
// that the taxonomy index has been recreated.
taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
taxoWriter.AddCategory(new FacetLabel("c")); // add a category so that commit will happen
taxoWriter.SetCommitData(new Dictionary <string, string>()
{
{ "just", "data" }
});
taxoWriter.Commit();
// verify taxoWriter.getCommitData()
Assert.IsTrue(taxoWriter.CommitData.ContainsKey(DirectoryTaxonomyWriter.INDEX_EPOCH), DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in taoxWriter.commitData");
taxoWriter.Dispose();
r = DirectoryReader.Open(dir);
readUserCommitData = r.IndexCommit.UserData;
Assert.IsTrue(readUserCommitData.ContainsKey(DirectoryTaxonomyWriter.INDEX_EPOCH), DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in taoxWriter.commitData");
r.Dispose();
dir.Dispose();
}
public virtual void TestAddToEmpty()
{
Directory dest = NewDirectory();
Directory src = NewDirectory();
DirectoryTaxonomyWriter srcTW = new DirectoryTaxonomyWriter(src);
srcTW.AddCategory(new FacetLabel("Author", "Rob Pike"));
srcTW.AddCategory(new FacetLabel("Aardvarks", "Bob"));
srcTW.Dispose();
DirectoryTaxonomyWriter destTW = new DirectoryTaxonomyWriter(dest);
OrdinalMap map = randomOrdinalMap();
destTW.AddTaxonomy(src, map);
destTW.Dispose();
validate(dest, src, map);
IOUtils.Close(dest, src);
}
private void doTestReadRecreatedTaxonomy(Random random, bool closeReader)
{
Directory dir = null;
ITaxonomyWriter tw = null;
TaxonomyReader tr = null;
// prepare a few categories
int n = 10;
FacetLabel[] cp = new FacetLabel[n];
for (int i = 0; i < n; i++)
{
cp[i] = new FacetLabel("a", Convert.ToString(i, CultureInfo.InvariantCulture));
}
try
{
dir = NewDirectory();
tw = new DirectoryTaxonomyWriter(dir);
tw.AddCategory(new FacetLabel("a"));
tw.Dispose();
tr = new DirectoryTaxonomyReader(dir);
int baseNumCategories = tr.Count;
for (int i = 0; i < n; i++)
{
int k = random.Next(n);
tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
for (int j = 0; j <= k; j++)
{
tw.AddCategory(cp[j]);
}
tw.Dispose();
if (closeReader)
{
tr.Dispose();
tr = new DirectoryTaxonomyReader(dir);
}
else
{
var newtr = TaxonomyReader.OpenIfChanged(tr);
Assert.IsNotNull(newtr);
tr.Dispose();
tr = newtr;
}
Assert.AreEqual(baseNumCategories + 1 + k, tr.Count, "Wrong #categories in taxonomy (i=" + i + ", k=" + k + ")");
}
}
finally
{
IOUtils.Dispose(tr, tw, dir);
}
}
public virtual void TestRollback()
{
// Verifies that if rollback is called, DTW is closed.
Directory dir = NewDirectory();
var dtw = new DirectoryTaxonomyWriter(dir);
dtw.AddCategory(new FacetLabel("a"));
dtw.Rollback();
try
{
dtw.AddCategory(new FacetLabel("a"));
Fail("should not have succeeded to add a category following rollback.");
}
catch (AlreadyClosedException)
{
// expected
}
dir.Dispose();
}
public virtual void TestReplaceTaxonomy()
{
Directory input = NewDirectory();
var taxoWriter = new DirectoryTaxonomyWriter(input);
int ordA = taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.Dispose();
Directory dir = NewDirectory();
taxoWriter = new DirectoryTaxonomyWriter(dir);
int ordB = taxoWriter.AddCategory(new FacetLabel("b"));
taxoWriter.AddCategory(new FacetLabel("c"));
taxoWriter.Commit();
long origEpoch = getEpoch(dir);
// replace the taxonomy with the input one
taxoWriter.ReplaceTaxonomy(input);
// LUCENE-4633: make sure that category "a" is not added again in any case
taxoWriter.AddTaxonomy(input, new MemoryOrdinalMap());
Assert.AreEqual(2, taxoWriter.Size, "no categories should have been added"); // root + 'a'
Assert.AreEqual(ordA, taxoWriter.AddCategory(new FacetLabel("a")), "category 'a' received new ordinal?");
// add the same category again -- it should not receive the same ordinal !
int newOrdB = taxoWriter.AddCategory(new FacetLabel("b"));
Assert.AreNotSame(ordB, newOrdB, "new ordinal cannot be the original ordinal");
Assert.AreEqual(2, newOrdB, "ordinal should have been 2 since only one category was added by replaceTaxonomy");
taxoWriter.Dispose();
long newEpoch = getEpoch(dir);
Assert.True(origEpoch < newEpoch, "index epoch should have been updated after replaceTaxonomy");
dir.Dispose();
input.Dispose();
}
public virtual void TestAddToEmpty()
{
Directory dest = NewDirectory();
Directory src = NewDirectory();
DirectoryTaxonomyWriter srcTW = new DirectoryTaxonomyWriter(src);
srcTW.AddCategory(new FacetLabel("Author", "Rob Pike"));
srcTW.AddCategory(new FacetLabel("Aardvarks", "Bob"));
srcTW.Dispose();
DirectoryTaxonomyWriter destTW = new DirectoryTaxonomyWriter(dest);
IOrdinalMap map = randomOrdinalMap();
destTW.AddTaxonomy(src, map);
destTW.Dispose();
validate(dest, src, map);
IOUtils.Dispose(dest, src);
}
public override void Run()
{
for (int i = 0; i < numCategories; i++)
{
try
{
destTW.AddCategory(new FacetLabel("a", Convert.ToString(i, CultureInfo.InvariantCulture)));
}
catch (IOException e)
{
// shouldn't happen - if it does, let the test fail on uncaught exception.
throw new Exception(e.ToString(), e);
}
}
}
public virtual void TestCloseTwice()
{
Directory dir = NewDirectory();
var ltw = new DirectoryTaxonomyWriter(dir);
ltw.AddCategory(new FacetLabel("a"));
ltw.Dispose();
var ltr = new DirectoryTaxonomyReader(dir);
ltr.Dispose();
ltr.Dispose(); // no exception should be thrown
dir.Dispose();
}
public virtual void TestConcurrency()
{
// tests that addTaxonomy and addCategory work in parallel
int numCategories = AtLeast(10000);
// build an input taxonomy index
Directory src = NewDirectory();
var tw = new DirectoryTaxonomyWriter(src);
for (int i = 0; i < numCategories; i++)
{
tw.AddCategory(new FacetLabel("a", Convert.ToString(i)));
}
tw.Dispose();
// now add the taxonomy to an empty taxonomy, while adding the categories
// again, in parallel -- in the end, no duplicate categories should exist.
Directory dest = NewDirectory();
var destTw = new DirectoryTaxonomyWriter(dest);
var t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw);
t.Start();
IOrdinalMap map = new MemoryOrdinalMap();
destTw.AddTaxonomy(src, map);
t.Join();
destTw.Dispose();
// now validate
var dtr = new DirectoryTaxonomyReader(dest);
// +2 to account for the root category + "a"
Assert.AreEqual(numCategories + 2, dtr.Count);
var categories = new JCG.HashSet <FacetLabel>();
for (int i = 1; i < dtr.Count; i++)
{
FacetLabel cat = dtr.GetPath(i);
Assert.True(categories.Add(cat), "category " + cat + " already existed");
}
dtr.Dispose();
IOUtils.Dispose(src, dest);
}
public virtual void TestCloseAfterIncRef()
{
Directory dir = NewDirectory();
var ltw = new DirectoryTaxonomyWriter(dir);
ltw.AddCategory(new FacetLabel("a"));
ltw.Dispose();
DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir);
ltr.IncRef();
ltr.Dispose();
// should not fail as we IncRef() before close
var tmpSie = ltr.Size;
ltr.DecRef();
dir.Dispose();
}
public override void Run()
{
Random random = Random;
while (numCats.DecrementAndGet() > 0)
{
string cat = Convert.ToString(random.Next(range));
try
{
tw.AddCategory(new FacetLabel("a", cat));
}
catch (IOException e)
{
throw new Exception(e.ToString(), e);
}
}
}
public override void Run()
{
Random random = Random;
while (numCats.DecrementAndGet() > 0)
{
string cat = Convert.ToString(random.Next(range), CultureInfo.InvariantCulture);
try
{
tw.AddCategory(new FacetLabel("a", cat));
}
catch (Exception e) when(e.IsIOException())
{
throw RuntimeException.Create(e);
}
}
}
public virtual void TestEnsureOpen()
{
// verifies that an exception is thrown if DTW was closed
Directory dir = NewDirectory();
DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir);
dtw.Dispose();
try
{
dtw.AddCategory(new FacetLabel("a"));
Fail("should not have succeeded to add a category following close.");
}
catch (AlreadyClosedException)
{
// expected
}
dir.Dispose();
}
public virtual void TestCommit()
{
// Verifies that nothing is committed to the underlying Directory, if
// commit() wasn't called.
Directory dir = NewDirectory();
var ltw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
Assert.False(DirectoryReader.IndexExists(dir));
ltw.Commit(); // first commit, so that an index will be created
ltw.AddCategory(new FacetLabel("a"));
IndexReader r = DirectoryReader.Open(dir);
Assert.AreEqual(1, r.NumDocs, "No categories should have been committed to the underlying directory");
r.Dispose();
ltw.Dispose();
dir.Dispose();
}
public virtual void TestOpenIfChangedReuseAfterRecreate()
{
// tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
Directory dir = NewDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
FacetLabel cp_a = new FacetLabel("a");
writer.AddCategory(cp_a);
writer.Dispose();
DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir);
// fill r1's caches
Assert.AreEqual(1, r1.GetOrdinal(cp_a));
Assert.AreEqual(cp_a, r1.GetPath(1));
// now recreate, add a different category
writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
FacetLabel cp_b = new FacetLabel("b");
writer.AddCategory(cp_b);
writer.Dispose();
DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
Assert.IsNotNull(r2);
// fill r2's caches
Assert.AreEqual(1, r2.GetOrdinal(cp_b));
Assert.AreEqual(cp_b, r2.GetPath(1));
// check that r1 doesn't see cp_b
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
Assert.AreEqual(cp_a, r1.GetPath(1));
// check that r2 doesn't see cp_a
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a));
Assert.AreEqual(cp_b, r2.GetPath(1));
r2.Dispose();
r1.Dispose();
dir.Dispose();
}
public virtual void TestCloseNoEmptyCommits()
{
// LUCENE-4972: DTW used to create empty commits even if no changes were made
Directory dir = NewDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.Commit();
long gen1 = SegmentInfos.GetLastCommitGeneration(dir);
taxoWriter.Dispose();
long gen2 = SegmentInfos.GetLastCommitGeneration(dir);
Assert.AreEqual(gen1, gen2, "empty commit should not have changed the index");
taxoWriter.Dispose();
dir.Dispose();
}
public virtual void TestAlreadyClosed()
{
Directory dir = NewDirectory();
var ltw = new DirectoryTaxonomyWriter(dir);
ltw.AddCategory(new FacetLabel("a"));
ltw.Dispose();
var ltr = new DirectoryTaxonomyReader(dir);
ltr.Dispose();
try
{
var tmpSize = ltr.Size;
Fail("An AlreadyClosedException should have been thrown here");
}
catch (AlreadyClosedException)
{
// good!
}
dir.Dispose();
}
public virtual void TestCloseAfterIncRef()
{
Directory dir = NewDirectory();
var ltw = new DirectoryTaxonomyWriter(dir);
ltw.AddCategory(new FacetLabel("a"));
ltw.Dispose();
DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir);
ltr.IncRef();
ltr.Dispose();
// should not fail as we IncRef() before close
var _ = ltr.Count;
ltr.DecRef();
dir.Dispose();
}
public virtual void TestAlreadyClosed()
{
Directory dir = NewDirectory();
var ltw = new DirectoryTaxonomyWriter(dir);
ltw.AddCategory(new FacetLabel("a"));
ltw.Dispose();
var ltr = new DirectoryTaxonomyReader(dir);
ltr.Dispose();
try
{
var _ = ltr.Count;
fail("An ObjectDisposedException should have been thrown here");
}
catch (ObjectDisposedException)
{
// good!
}
dir.Dispose();
}
public virtual void TestReplaceTaxoWithLargeTaxonomy()
{
var srcTaxoDir = NewDirectory();
var targetTaxoDir = NewDirectory();
// build source, large, taxonomy
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(srcTaxoDir);
int ord = taxoWriter.AddCategory(new FacetLabel("A", "1", "1", "1", "1", "1", "1"));
taxoWriter.Dispose();
taxoWriter = new DirectoryTaxonomyWriter(targetTaxoDir);
int ordinal = taxoWriter.AddCategory(new FacetLabel("B", "1"));
Assert.AreEqual(1, taxoWriter.GetParent(ordinal)); // call getParent to initialize taxoArrays
taxoWriter.Commit();
taxoWriter.ReplaceTaxonomy(srcTaxoDir);
Assert.AreEqual(ord - 1, taxoWriter.GetParent(ord));
taxoWriter.Dispose();
srcTaxoDir.Dispose();
targetTaxoDir.Dispose();
}
public virtual void TestOpenIfChangedResult()
{
Directory dir = null;
DirectoryTaxonomyWriter ltw = null;
DirectoryTaxonomyReader ltr = null;
try
{
dir = NewDirectory();
ltw = new DirectoryTaxonomyWriter(dir);
ltw.AddCategory(new FacetLabel("a"));
ltw.Commit();
ltr = new DirectoryTaxonomyReader(dir);
Assert.Null(TaxonomyReader.OpenIfChanged(ltr), "Nothing has changed");
ltw.AddCategory(new FacetLabel("b"));
ltw.Commit();
DirectoryTaxonomyReader newtr = TaxonomyReader.OpenIfChanged(ltr);
Assert.NotNull(newtr, "changes were committed");
Assert.Null(TaxonomyReader.OpenIfChanged(newtr), "Nothing has changed");
(newtr).Dispose();
}
finally
{
IOUtils.Close(ltw, ltr, dir);
}
}
public virtual void TestSimple()
{
Directory dest = NewDirectory();
var tw1 = new DirectoryTaxonomyWriter(dest);
tw1.AddCategory(new FacetLabel("Author", "Mark Twain"));
tw1.AddCategory(new FacetLabel("Animals", "Dog"));
tw1.AddCategory(new FacetLabel("Author", "Rob Pike"));
Directory src = NewDirectory();
var tw2 = new DirectoryTaxonomyWriter(src);
tw2.AddCategory(new FacetLabel("Author", "Rob Pike"));
tw2.AddCategory(new FacetLabel("Aardvarks", "Bob"));
tw2.Dispose();
OrdinalMap map = randomOrdinalMap();
tw1.AddTaxonomy(src, map);
tw1.Dispose();
validate(dest, src, map);
IOUtils.Close(dest, src);
}
public virtual void TestOpenIfChangedReuse()
{
// test the reuse of data from the old DTR instance
foreach (bool nrt in new bool[] { false, true })
{
Directory dir = NewDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
FacetLabel cp_a = new FacetLabel("a");
writer.AddCategory(cp_a);
if (!nrt)
{
writer.Commit();
}
DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
// fill r1's caches
Assert.AreEqual(1, r1.GetOrdinal(cp_a));
Assert.AreEqual(cp_a, r1.GetPath(1));
FacetLabel cp_b = new FacetLabel("b");
writer.AddCategory(cp_b);
if (!nrt)
{
writer.Commit();
}
DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
Assert.NotNull(r2);
// add r2's categories to the caches
Assert.AreEqual(2, r2.GetOrdinal(cp_b));
Assert.AreEqual(cp_b, r2.GetPath(2));
// check that r1 doesn't see cp_b
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
Assert.Null(r1.GetPath(2));
(r1).Dispose();
(r2).Dispose();
writer.Dispose();
dir.Dispose();
}
}
public virtual void TestOpenIfChangedReuseAfterRecreate()
{
// tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
Directory dir = NewDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
FacetLabel cp_a = new FacetLabel("a");
writer.AddCategory(cp_a);
writer.Dispose();
DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir);
// fill r1's caches
Assert.AreEqual(1, r1.GetOrdinal(cp_a));
Assert.AreEqual(cp_a, r1.GetPath(1));
// now recreate, add a different category
writer = new DirectoryTaxonomyWriter(dir, IndexWriterConfig.OpenMode_e.CREATE);
FacetLabel cp_b = new FacetLabel("b");
writer.AddCategory(cp_b);
writer.Dispose();
DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
Assert.NotNull(r2);
// fill r2's caches
Assert.AreEqual(1, r2.GetOrdinal(cp_b));
Assert.AreEqual(cp_b, r2.GetPath(1));
// check that r1 doesn't see cp_b
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
Assert.AreEqual(cp_a, r1.GetPath(1));
// check that r2 doesn't see cp_a
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a));
Assert.AreEqual(cp_b, r2.GetPath(1));
(r2).Dispose();
(r1).Dispose();
dir.Dispose();
}
public virtual void TestCloseTwice()
{
Directory dir = NewDirectory();
var ltw = new DirectoryTaxonomyWriter(dir);
ltw.AddCategory(new FacetLabel("a"));
ltw.Dispose();
var ltr = new DirectoryTaxonomyReader(dir);
(ltr).Dispose();
(ltr).Dispose(); // no exception should be thrown
dir.Dispose();
}
public virtual void TestOpenIfChangedReplaceTaxonomy()
{
// test openIfChanged when replaceTaxonomy is called, which is equivalent to recreate
// only can work with NRT as well
Directory src = NewDirectory();
DirectoryTaxonomyWriter w = new DirectoryTaxonomyWriter(src);
FacetLabel cp_b = new FacetLabel("b");
w.AddCategory(cp_b);
w.Dispose();
foreach (bool nrt in new bool[] { false, true })
{
Directory dir = NewDirectory();
var writer = new DirectoryTaxonomyWriter(dir);
FacetLabel cp_a = new FacetLabel("a");
writer.AddCategory(cp_a);
if (!nrt)
{
writer.Commit();
}
DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
// fill r1's caches
Assert.AreEqual(1, r1.GetOrdinal(cp_a));
Assert.AreEqual(cp_a, r1.GetPath(1));
// now replace taxonomy
writer.ReplaceTaxonomy(src);
if (!nrt)
{
writer.Commit();
}
DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
Assert.NotNull(r2);
// fill r2's caches
Assert.AreEqual(1, r2.GetOrdinal(cp_b));
Assert.AreEqual(cp_b, r2.GetPath(1));
// check that r1 doesn't see cp_b
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
Assert.AreEqual(cp_a, r1.GetPath(1));
// check that r2 doesn't see cp_a
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a));
Assert.AreEqual(cp_b, r2.GetPath(1));
(r2).Dispose();
(r1).Dispose();
writer.Dispose();
dir.Dispose();
}
src.Dispose();
}
public virtual void TestReplaceTaxoWithLargeTaxonomy()
{
var srcTaxoDir = NewDirectory();
var targetTaxoDir = NewDirectory();
// build source, large, taxonomy
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(srcTaxoDir);
int ord = taxoWriter.AddCategory(new FacetLabel("A", "1", "1", "1", "1", "1", "1"));
taxoWriter.Dispose();
taxoWriter = new DirectoryTaxonomyWriter(targetTaxoDir);
int ordinal = taxoWriter.AddCategory(new FacetLabel("B", "1"));
Assert.AreEqual(1, taxoWriter.GetParent(ordinal)); // call getParent to initialize taxoArrays
taxoWriter.Commit();
taxoWriter.ReplaceTaxonomy(srcTaxoDir);
Assert.AreEqual(ord - 1, taxoWriter.GetParent(ord));
taxoWriter.Dispose();
srcTaxoDir.Dispose();
targetTaxoDir.Dispose();
}
public virtual void TestOpenIfChangedAndRefCount()
{
Directory dir = new RAMDirectory(); // no need for random directories here
var taxoWriter = new DirectoryTaxonomyWriter(dir);
taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.Commit();
var taxoReader = new DirectoryTaxonomyReader(dir);
Assert.AreEqual(1, taxoReader.RefCount, "wrong refCount");
taxoReader.IncRef();
Assert.AreEqual(2, taxoReader.RefCount, "wrong refCount");
taxoWriter.AddCategory(new FacetLabel("a", "b"));
taxoWriter.Commit();
var newtr = TaxonomyReader.OpenIfChanged(taxoReader);
Assert.NotNull(newtr);
taxoReader.Dispose();
taxoReader = newtr;
Assert.AreEqual(1, taxoReader.RefCount, "wrong refCount");
taxoWriter.Dispose();
taxoReader.Dispose();
dir.Dispose();
}
public virtual void TestGetChildren()
{
Directory dir = NewDirectory();
var taxoWriter = new DirectoryTaxonomyWriter(dir);
int numCategories = AtLeast(10);
int numA = 0, numB = 0;
Random random = Random;
// add the two categories for which we'll also add children (so asserts are simpler)
taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.AddCategory(new FacetLabel("b"));
for (int i = 0; i < numCategories; i++)
{
if (random.NextBoolean())
{
taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i, CultureInfo.InvariantCulture)));
++numA;
}
else
{
taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i, CultureInfo.InvariantCulture)));
++numB;
}
}
// add category with no children
taxoWriter.AddCategory(new FacetLabel("c"));
taxoWriter.Dispose();
var taxoReader = new DirectoryTaxonomyReader(dir);
// non existing category
TaxonomyReader.ChildrenEnumerator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid")));
Assert.AreEqual(false, it.MoveNext());
// a category with no children
it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c")));
Assert.AreEqual(false, it.MoveNext());
// arbitrary negative ordinal
it = taxoReader.GetChildren(-2);
Assert.AreEqual(false, it.MoveNext());
// root's children
var roots = new JCG.HashSet <string> {
"a", "b", "c"
};
it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL);
while (roots.Count > 0)
{
it.MoveNext();
FacetLabel root = taxoReader.GetPath(it.Current);
Assert.AreEqual(1, root.Length);
Assert.IsTrue(roots.Remove(root.Components[0]));
}
Assert.AreEqual(false, it.MoveNext());
for (int i = 0; i < 2; i++)
{
FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b");
int ordinal = taxoReader.GetOrdinal(cp);
it = taxoReader.GetChildren(ordinal);
int numChildren = 0;
int child;
while (it.MoveNext())
{
child = it.Current;
FacetLabel path = taxoReader.GetPath(child);
Assert.AreEqual(2, path.Length);
Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b");
++numChildren;
}
int expected = i == 0 ? numA : numB;
Assert.AreEqual(expected, numChildren, "invalid num children");
}
taxoReader.Dispose();
dir.Dispose();
}
private void doTestReadRecreatedTaxonomy(Random random, bool closeReader)
{
Directory dir = null;
TaxonomyWriter tw = null;
TaxonomyReader tr = null;
// prepare a few categories
int n = 10;
FacetLabel[] cp = new FacetLabel[n];
for (int i = 0; i < n; i++)
{
cp[i] = new FacetLabel("a", Convert.ToString(i));
}
try
{
dir = NewDirectory();
tw = new DirectoryTaxonomyWriter(dir);
tw.AddCategory(new FacetLabel("a"));
tw.Dispose();
tr = new DirectoryTaxonomyReader(dir);
int baseNumCategories = tr.Size;
for (int i = 0; i < n; i++)
{
int k = random.Next(n);
tw = new DirectoryTaxonomyWriter(dir, IndexWriterConfig.OpenMode_e.CREATE);
for (int j = 0; j <= k; j++)
{
tw.AddCategory(cp[j]);
}
tw.Dispose();
if (closeReader)
{
tr.Dispose(true);
tr = new DirectoryTaxonomyReader(dir);
}
else
{
var newtr = TaxonomyReader.OpenIfChanged(tr);
Assert.NotNull(newtr);
tr.Dispose(true);
tr = newtr;
}
Assert.AreEqual(baseNumCategories + 1 + k, tr.Size, "Wrong #categories in taxonomy (i=" + i + ", k=" + k + ")");
}
}
finally
{
IOUtils.Close(tr as DirectoryTaxonomyReader, tw, dir);
}
}
public virtual void TestReplaceTaxonomy()
{
Directory input = NewDirectory();
var taxoWriter = new DirectoryTaxonomyWriter(input);
int ordA = taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.Dispose();
Directory dir = NewDirectory();
taxoWriter = new DirectoryTaxonomyWriter(dir);
int ordB = taxoWriter.AddCategory(new FacetLabel("b"));
taxoWriter.AddCategory(new FacetLabel("c"));
taxoWriter.Commit();
long origEpoch = getEpoch(dir);
// replace the taxonomy with the input one
taxoWriter.ReplaceTaxonomy(input);
// LUCENE-4633: make sure that category "a" is not added again in any case
taxoWriter.AddTaxonomy(input, new MemoryOrdinalMap());
Assert.AreEqual(2, taxoWriter.Count, "no categories should have been added"); // root + 'a'
Assert.AreEqual(ordA, taxoWriter.AddCategory(new FacetLabel("a")), "category 'a' received new ordinal?");
// add the same category again -- it should not receive the same ordinal !
int newOrdB = taxoWriter.AddCategory(new FacetLabel("b"));
Assert.AreNotSame(ordB, newOrdB, "new ordinal cannot be the original ordinal");
Assert.AreEqual(2, newOrdB, "ordinal should have been 2 since only one category was added by replaceTaxonomy");
taxoWriter.Dispose();
long newEpoch = getEpoch(dir);
Assert.True(origEpoch < newEpoch, "index epoch should have been updated after replaceTaxonomy");
dir.Dispose();
input.Dispose();
}
private void TouchTaxo(DirectoryTaxonomyWriter taxoWriter, FacetLabel cp)
{
taxoWriter.AddCategory(cp);
taxoWriter.CommitData = new Dictionary<string, string>()
{
{"just", "data"}
};
taxoWriter.Commit();
}
public virtual void TestConcurrency()
{
// tests that addTaxonomy and addCategory work in parallel
int numCategories = AtLeast(10000);
// build an input taxonomy index
Directory src = NewDirectory();
var tw = new DirectoryTaxonomyWriter(src);
for (int i = 0; i < numCategories; i++)
{
tw.AddCategory(new FacetLabel("a", Convert.ToString(i)));
}
tw.Dispose();
// now add the taxonomy to an empty taxonomy, while adding the categories
// again, in parallel -- in the end, no duplicate categories should exist.
Directory dest = NewDirectory();
var destTw = new DirectoryTaxonomyWriter(dest);
ThreadClass t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw);
t.Start();
OrdinalMap map = new MemoryOrdinalMap();
destTw.AddTaxonomy(src, map);
t.Join();
destTw.Dispose();
// now validate
var dtr = new DirectoryTaxonomyReader(dest);
// +2 to account for the root category + "a"
Assert.AreEqual(numCategories + 2, dtr.Size);
var categories = new HashSet<FacetLabel>();
for (int i = 1; i < dtr.Size; i++)
{
FacetLabel cat = dtr.GetPath(i);
Assert.True(categories.Add(cat), "category " + cat + " already existed");
}
dtr.Dispose();
IOUtils.Close(src, dest);
}
public virtual void TestCommit()
{
// Verifies that nothing is committed to the underlying Directory, if
// commit() wasn't called.
Directory dir = NewDirectory();
var ltw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
Assert.False(DirectoryReader.IndexExists(dir));
ltw.Commit(); // first commit, so that an index will be created
ltw.AddCategory(new FacetLabel("a"));
IndexReader r = DirectoryReader.Open(dir);
Assert.AreEqual(1, r.NumDocs, "No categories should have been committed to the underlying directory");
r.Dispose();
ltw.Dispose();
dir.Dispose();
}
public virtual void TestCommitUserData()
{
// Verifies taxonomy commit data
Directory dir = NewDirectory();
var taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.AddCategory(new FacetLabel("b"));
IDictionary<string, string> userCommitData = new Dictionary<string, string>();
userCommitData["testing"] = "1 2 3";
taxoWriter.CommitData = userCommitData;
taxoWriter.Dispose();
var r = DirectoryReader.Open(dir);
Assert.AreEqual(3, r.NumDocs, "2 categories plus root should have been committed to the underlying directory");
var readUserCommitData = r.IndexCommit.UserData;
Assert.True("1 2 3".Equals(readUserCommitData["testing"]), "wrong value extracted from commit data");
Assert.NotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in commitData", readUserCommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]);
r.Dispose();
// open DirTaxoWriter again and commit, INDEX_EPOCH should still exist
// in the commit data, otherwise DirTaxoReader.refresh() might not detect
// that the taxonomy index has been recreated.
taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
taxoWriter.AddCategory(new FacetLabel("c")); // add a category so that commit will happen
taxoWriter.CommitData = new Dictionary<string, string>()
{
{"just", "data"}
};
taxoWriter.Commit();
// verify taxoWriter.getCommitData()
Assert.NotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in taoxWriter.commitData", taxoWriter.CommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]);
taxoWriter.Dispose();
r = DirectoryReader.Open(dir);
readUserCommitData = r.IndexCommit.UserData;
Assert.NotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in commitData", readUserCommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]);
r.Dispose();
dir.Dispose();
}
public virtual void TestReaderFreshness()
{
// ensures that the internal index reader is always kept fresh. Previously,
// this simple scenario failed, if the cache just evicted the category that
// is being added.
Directory dir = NewDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
int o1 = taxoWriter.AddCategory(new FacetLabel("a"));
int o2 = taxoWriter.AddCategory(new FacetLabel("a"));
Assert.True(o1 == o2, "ordinal for same category that is added twice should be the same !");
taxoWriter.Dispose();
dir.Dispose();
}
public virtual void TestRollback()
{
// Verifies that if rollback is called, DTW is closed.
Directory dir = NewDirectory();
var dtw = new DirectoryTaxonomyWriter(dir);
dtw.AddCategory(new FacetLabel("a"));
dtw.Rollback();
try
{
dtw.AddCategory(new FacetLabel("a"));
Fail("should not have succeeded to add a category following rollback.");
}
catch (AlreadyClosedException)
{
// expected
}
dir.Dispose();
}
public virtual void TestPrepareCommitNoEmptyCommits()
{
// LUCENE-4972: DTW used to create empty commits even if no changes were made
Directory dir = NewDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.PrepareCommit();
taxoWriter.Commit();
long gen1 = SegmentInfos.GetLastCommitGeneration(dir);
taxoWriter.PrepareCommit();
taxoWriter.Commit();
long gen2 = SegmentInfos.GetLastCommitGeneration(dir);
Assert.AreEqual(gen1, gen2, "empty commit should not have changed the index");
taxoWriter.Dispose();
dir.Dispose();
}
public virtual void TestHugeLabel()
{
Directory indexDir = NewDirectory(), taxoDir = NewDirectory();
IndexWriter indexWriter = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, new Cl2oTaxonomyWriterCache(2, 1f, 1));
FacetsConfig config = new FacetsConfig();
// Add one huge label:
string bigs = null;
int ordinal = -1;
int len = FacetLabel.MAX_CATEGORY_PATH_LENGTH - 4; // for the dimension and separator
bigs = TestUtil.RandomSimpleString(Random(), len, len);
FacetField ff = new FacetField("dim", bigs);
FacetLabel cp = new FacetLabel("dim", bigs);
ordinal = taxoWriter.AddCategory(cp);
Document doc = new Document();
doc.Add(ff);
indexWriter.AddDocument(config.Build(taxoWriter, doc));
// Add tiny ones to cause a re-hash
for (int i = 0; i < 3; i++)
{
string s = TestUtil.RandomSimpleString(Random(), 1, 10);
taxoWriter.AddCategory(new FacetLabel("dim", s));
doc = new Document();
doc.Add(new FacetField("dim", s));
indexWriter.AddDocument(config.Build(taxoWriter, doc));
}
// when too large components were allowed to be added, this resulted in a new added category
Assert.AreEqual(ordinal, taxoWriter.AddCategory(cp));
IOUtils.Close(indexWriter, taxoWriter);
DirectoryReader indexReader = DirectoryReader.Open(indexDir);
var taxoReader = new DirectoryTaxonomyReader(taxoDir);
IndexSearcher searcher = new IndexSearcher(indexReader);
DrillDownQuery ddq = new DrillDownQuery(new FacetsConfig());
ddq.Add("dim", bigs);
Assert.AreEqual(1, searcher.Search(ddq, 10).TotalHits);
IOUtils.Close(indexReader, taxoReader, indexDir, taxoDir);
}
public virtual void TestEnsureOpen()
{
// verifies that an exception is thrown if DTW was closed
Directory dir = NewDirectory();
DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir);
dtw.Dispose();
try
{
dtw.AddCategory(new FacetLabel("a"));
Fail("should not have succeeded to add a category following close.");
}
catch (AlreadyClosedException)
{
// expected
}
dir.Dispose();
}
public virtual void TestGetChildren()
{
Directory dir = NewDirectory();
var taxoWriter = new DirectoryTaxonomyWriter(dir);
int numCategories = AtLeast(10);
int numA = 0, numB = 0;
Random random = Random();
// add the two categories for which we'll also add children (so asserts are simpler)
taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.AddCategory(new FacetLabel("b"));
for (int i = 0; i < numCategories; i++)
{
if (random.NextBoolean())
{
taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i)));
++numA;
}
else
{
taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i)));
++numB;
}
}
// add category with no children
taxoWriter.AddCategory(new FacetLabel("c"));
taxoWriter.Dispose();
var taxoReader = new DirectoryTaxonomyReader(dir);
// non existing category
TaxonomyReader.ChildrenIterator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid")));
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
// a category with no children
it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c")));
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
// arbitrary negative ordinal
it = taxoReader.GetChildren(-2);
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
// root's children
var roots = new HashSet<string>(Arrays.AsList("a", "b", "c"));
it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL);
while (roots.Count > 0)
{
FacetLabel root = taxoReader.GetPath(it.Next());
Assert.AreEqual(1, root.Length);
Assert.True(roots.Remove(root.Components[0]));
}
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
for (int i = 0; i < 2; i++)
{
FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b");
int ordinal = taxoReader.GetOrdinal(cp);
it = taxoReader.GetChildren(ordinal);
int numChildren = 0;
int child;
while ((child = it.Next()) != TaxonomyReader.INVALID_ORDINAL)
{
FacetLabel path = taxoReader.GetPath(child);
Assert.AreEqual(2, path.Length);
Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b");
++numChildren;
}
int expected = i == 0 ? numA : numB;
Assert.AreEqual(expected, numChildren, "invalid num children");
}
taxoReader.Dispose();
dir.Dispose();
}
