public virtual void TestOpenIfChangedReplaceTaxonomy()
{
// test openIfChanged when replaceTaxonomy is called, which is equivalent to recreate
// only can work with NRT as well
Directory src = NewDirectory();
DirectoryTaxonomyWriter w = new DirectoryTaxonomyWriter(src);
FacetLabel cp_b = new FacetLabel("b");
w.AddCategory(cp_b);
w.Dispose();
foreach (bool nrt in new bool[] { false, true })
{
Directory dir = NewDirectory();
var writer = new DirectoryTaxonomyWriter(dir);
FacetLabel cp_a = new FacetLabel("a");
writer.AddCategory(cp_a);
if (!nrt)
{
writer.Commit();
}
DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
// fill r1's caches
Assert.AreEqual(1, r1.GetOrdinal(cp_a));
Assert.AreEqual(cp_a, r1.GetPath(1));
// now replace taxonomy
writer.ReplaceTaxonomy(src);
if (!nrt)
{
writer.Commit();
}
DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
Assert.IsNotNull(r2);
// fill r2's caches
Assert.AreEqual(1, r2.GetOrdinal(cp_b));
Assert.AreEqual(cp_b, r2.GetPath(1));
// check that r1 doesn't see cp_b
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
Assert.AreEqual(cp_a, r1.GetPath(1));
// check that r2 doesn't see cp_a
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a));
Assert.AreEqual(cp_b, r2.GetPath(1));
r2.Dispose();
r1.Dispose();
writer.Dispose();
dir.Dispose();
}
src.Dispose();
}
private void validate(Directory dest, Directory src, IOrdinalMap ordMap)
{
using (var destTr = new DirectoryTaxonomyReader(dest))
{
int destSize = destTr.Count;
using (var srcTR = new DirectoryTaxonomyReader(src))
{
var map = ordMap.GetMap();
// validate taxo sizes
int srcSize = srcTR.Count;
Assert.True(destSize >= srcSize, "destination taxonomy expected to be larger than source; dest=" + destSize + " src=" + srcSize);
// validate that all source categories exist in destination, and their
// ordinals are as expected.
for (int j = 1; j < srcSize; j++)
{
FacetLabel cp = srcTR.GetPath(j);
int destOrdinal = destTr.GetOrdinal(cp);
Assert.True(destOrdinal > 0, cp + " not found in destination");
Assert.AreEqual(destOrdinal, map[j]);
}
}
}
}
public virtual void TestOpenIfChangedReuse()
{
// test the reuse of data from the old DTR instance
foreach (bool nrt in new bool[] { false, true })
{
Directory dir = NewDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
FacetLabel cp_a = new FacetLabel("a");
writer.AddCategory(cp_a);
if (!nrt)
{
writer.Commit();
}
DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
// fill r1's caches
Assert.AreEqual(1, r1.GetOrdinal(cp_a));
Assert.AreEqual(cp_a, r1.GetPath(1));
FacetLabel cp_b = new FacetLabel("b");
writer.AddCategory(cp_b);
if (!nrt)
{
writer.Commit();
}
DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
Assert.IsNotNull(r2);
// add r2's categories to the caches
Assert.AreEqual(2, r2.GetOrdinal(cp_b));
Assert.AreEqual(cp_b, r2.GetPath(2));
// check that r1 doesn't see cp_b
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
Assert.IsNull(r1.GetPath(2));
r1.Dispose();
r2.Dispose();
writer.Dispose();
dir.Dispose();
}
}
public virtual void TestOpenIfChangedReuseAfterRecreate()
{
// tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
Directory dir = NewDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
FacetLabel cp_a = new FacetLabel("a");
writer.AddCategory(cp_a);
writer.Dispose();
DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir);
// fill r1's caches
Assert.AreEqual(1, r1.GetOrdinal(cp_a));
Assert.AreEqual(cp_a, r1.GetPath(1));
// now recreate, add a different category
writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
FacetLabel cp_b = new FacetLabel("b");
writer.AddCategory(cp_b);
writer.Dispose();
DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
Assert.IsNotNull(r2);
// fill r2's caches
Assert.AreEqual(1, r2.GetOrdinal(cp_b));
Assert.AreEqual(cp_b, r2.GetPath(1));
// check that r1 doesn't see cp_b
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
Assert.AreEqual(cp_a, r1.GetPath(1));
// check that r2 doesn't see cp_a
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a));
Assert.AreEqual(cp_b, r2.GetPath(1));
r2.Dispose();
r1.Dispose();
dir.Dispose();
}
public virtual void TestConcurrency()
{
// tests that addTaxonomy and addCategory work in parallel
int numCategories = AtLeast(10000);
// build an input taxonomy index
Directory src = NewDirectory();
var tw = new DirectoryTaxonomyWriter(src);
for (int i = 0; i < numCategories; i++)
{
tw.AddCategory(new FacetLabel("a", Convert.ToString(i)));
}
tw.Dispose();
// now add the taxonomy to an empty taxonomy, while adding the categories
// again, in parallel -- in the end, no duplicate categories should exist.
Directory dest = NewDirectory();
var destTw = new DirectoryTaxonomyWriter(dest);
var t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw);
t.Start();
IOrdinalMap map = new MemoryOrdinalMap();
destTw.AddTaxonomy(src, map);
t.Join();
destTw.Dispose();
// now validate
var dtr = new DirectoryTaxonomyReader(dest);
// +2 to account for the root category + "a"
Assert.AreEqual(numCategories + 2, dtr.Count);
var categories = new JCG.HashSet <FacetLabel>();
for (int i = 1; i < dtr.Count; i++)
{
FacetLabel cat = dtr.GetPath(i);
Assert.True(categories.Add(cat), "category " + cat + " already existed");
}
dtr.Dispose();
IOUtils.Dispose(src, dest);
}
public virtual void TestGetChildren()
{
Directory dir = NewDirectory();
var taxoWriter = new DirectoryTaxonomyWriter(dir);
int numCategories = AtLeast(10);
int numA = 0, numB = 0;
Random random = Random;
// add the two categories for which we'll also add children (so asserts are simpler)
taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.AddCategory(new FacetLabel("b"));
for (int i = 0; i < numCategories; i++)
{
if (random.NextBoolean())
{
taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i, CultureInfo.InvariantCulture)));
++numA;
}
else
{
taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i, CultureInfo.InvariantCulture)));
++numB;
}
}
// add category with no children
taxoWriter.AddCategory(new FacetLabel("c"));
taxoWriter.Dispose();
var taxoReader = new DirectoryTaxonomyReader(dir);
// non existing category
TaxonomyReader.ChildrenEnumerator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid")));
Assert.AreEqual(false, it.MoveNext());
// a category with no children
it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c")));
Assert.AreEqual(false, it.MoveNext());
// arbitrary negative ordinal
it = taxoReader.GetChildren(-2);
Assert.AreEqual(false, it.MoveNext());
// root's children
var roots = new JCG.HashSet <string> {
"a", "b", "c"
};
it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL);
while (roots.Count > 0)
{
it.MoveNext();
FacetLabel root = taxoReader.GetPath(it.Current);
Assert.AreEqual(1, root.Length);
Assert.IsTrue(roots.Remove(root.Components[0]));
}
Assert.AreEqual(false, it.MoveNext());
for (int i = 0; i < 2; i++)
{
FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b");
int ordinal = taxoReader.GetOrdinal(cp);
it = taxoReader.GetChildren(ordinal);
int numChildren = 0;
int child;
while (it.MoveNext())
{
child = it.Current;
FacetLabel path = taxoReader.GetPath(child);
Assert.AreEqual(2, path.Length);
Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b");
++numChildren;
}
int expected = i == 0 ? numA : numB;
Assert.AreEqual(expected, numChildren, "invalid num children");
}
taxoReader.Dispose();
dir.Dispose();
}
public virtual void TestConcurrency()
{
// tests that addTaxonomy and addCategory work in parallel
int numCategories = AtLeast(10000);
// build an input taxonomy index
Directory src = NewDirectory();
var tw = new DirectoryTaxonomyWriter(src);
for (int i = 0; i < numCategories; i++)
{
tw.AddCategory(new FacetLabel("a", Convert.ToString(i)));
}
tw.Dispose();
// now add the taxonomy to an empty taxonomy, while adding the categories
// again, in parallel -- in the end, no duplicate categories should exist.
Directory dest = NewDirectory();
var destTw = new DirectoryTaxonomyWriter(dest);
ThreadClass t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw);
t.Start();
OrdinalMap map = new MemoryOrdinalMap();
destTw.AddTaxonomy(src, map);
t.Join();
destTw.Dispose();
// now validate
var dtr = new DirectoryTaxonomyReader(dest);
// +2 to account for the root category + "a"
Assert.AreEqual(numCategories + 2, dtr.Size);
var categories = new HashSet<FacetLabel>();
for (int i = 1; i < dtr.Size; i++)
{
FacetLabel cat = dtr.GetPath(i);
Assert.True(categories.Add(cat), "category " + cat + " already existed");
}
dtr.Dispose();
IOUtils.Close(src, dest);
}
private void validate(Directory dest, Directory src, OrdinalMap ordMap)
{
var destTr = new DirectoryTaxonomyReader(dest);
try
{
int destSize = destTr.Size;
var srcTR = new DirectoryTaxonomyReader(src);
try
{
var map = ordMap.Map;
// validate taxo sizes
int srcSize = srcTR.Size;
Assert.True(destSize >= srcSize, "destination taxonomy expected to be larger than source; dest=" + destSize + " src=" + srcSize);
// validate that all source categories exist in destination, and their
// ordinals are as expected.
for (int j = 1; j < srcSize; j++)
{
FacetLabel cp = srcTR.GetPath(j);
int destOrdinal = destTr.GetOrdinal(cp);
Assert.True(destOrdinal > 0, cp + " not found in destination");
Assert.AreEqual(destOrdinal, map[j]);
}
}
finally
{
((TaxonomyReader)srcTR).Dispose(true);
}
}
finally
{
((TaxonomyReader)destTr).Dispose(true);
}
}
public virtual void TestOpenIfChangedReuseAfterRecreate()
{
// tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
Directory dir = NewDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
FacetLabel cp_a = new FacetLabel("a");
writer.AddCategory(cp_a);
writer.Dispose();
DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir);
// fill r1's caches
Assert.AreEqual(1, r1.GetOrdinal(cp_a));
Assert.AreEqual(cp_a, r1.GetPath(1));
// now recreate, add a different category
writer = new DirectoryTaxonomyWriter(dir, IndexWriterConfig.OpenMode_e.CREATE);
FacetLabel cp_b = new FacetLabel("b");
writer.AddCategory(cp_b);
writer.Dispose();
DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1);
Assert.NotNull(r2);
// fill r2's caches
Assert.AreEqual(1, r2.GetOrdinal(cp_b));
Assert.AreEqual(cp_b, r2.GetPath(1));
// check that r1 doesn't see cp_b
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b));
Assert.AreEqual(cp_a, r1.GetPath(1));
// check that r2 doesn't see cp_a
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a));
Assert.AreEqual(cp_b, r2.GetPath(1));
(r2).Dispose();
(r1).Dispose();
dir.Dispose();
}
public virtual void TestGetChildren()
{
Directory dir = NewDirectory();
var taxoWriter = new DirectoryTaxonomyWriter(dir);
int numCategories = AtLeast(10);
int numA = 0, numB = 0;
Random random = Random();
// add the two categories for which we'll also add children (so asserts are simpler)
taxoWriter.AddCategory(new FacetLabel("a"));
taxoWriter.AddCategory(new FacetLabel("b"));
for (int i = 0; i < numCategories; i++)
{
if (random.NextBoolean())
{
taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i)));
++numA;
}
else
{
taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i)));
++numB;
}
}
// add category with no children
taxoWriter.AddCategory(new FacetLabel("c"));
taxoWriter.Dispose();
var taxoReader = new DirectoryTaxonomyReader(dir);
// non existing category
TaxonomyReader.ChildrenIterator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid")));
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
// a category with no children
it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c")));
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
// arbitrary negative ordinal
it = taxoReader.GetChildren(-2);
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
// root's children
var roots = new HashSet<string>(Arrays.AsList("a", "b", "c"));
it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL);
while (roots.Count > 0)
{
FacetLabel root = taxoReader.GetPath(it.Next());
Assert.AreEqual(1, root.Length);
Assert.True(roots.Remove(root.Components[0]));
}
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next());
for (int i = 0; i < 2; i++)
{
FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b");
int ordinal = taxoReader.GetOrdinal(cp);
it = taxoReader.GetChildren(ordinal);
int numChildren = 0;
int child;
while ((child = it.Next()) != TaxonomyReader.INVALID_ORDINAL)
{
FacetLabel path = taxoReader.GetPath(child);
Assert.AreEqual(2, path.Length);
Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b");
++numChildren;
}
int expected = i == 0 ? numA : numB;
Assert.AreEqual(expected, numChildren, "invalid num children");
}
taxoReader.Dispose();
dir.Dispose();
}
