public virtual void TestOpenIfChangedReplaceTaxonomy() { // test openIfChanged when replaceTaxonomy is called, which is equivalent to recreate // only can work with NRT as well Directory src = NewDirectory(); DirectoryTaxonomyWriter w = new DirectoryTaxonomyWriter(src); FacetLabel cp_b = new FacetLabel("b"); w.AddCategory(cp_b); w.Dispose(); foreach (bool nrt in new bool[] { false, true }) { Directory dir = NewDirectory(); var writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); // now replace taxonomy writer.ReplaceTaxonomy(src); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.IsNotNull(r2); // fill r2's caches Assert.AreEqual(1, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(1)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.AreEqual(cp_a, r1.GetPath(1)); // check that r2 doesn't see cp_a Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a)); Assert.AreEqual(cp_b, r2.GetPath(1)); r2.Dispose(); r1.Dispose(); writer.Dispose(); dir.Dispose(); } src.Dispose(); }
private void validate(Directory dest, Directory src, IOrdinalMap ordMap) { using (var destTr = new DirectoryTaxonomyReader(dest)) { int destSize = destTr.Count; using (var srcTR = new DirectoryTaxonomyReader(src)) { var map = ordMap.GetMap(); // validate taxo sizes int srcSize = srcTR.Count; Assert.True(destSize >= srcSize, "destination taxonomy expected to be larger than source; dest=" + destSize + " src=" + srcSize); // validate that all source categories exist in destination, and their // ordinals are as expected. for (int j = 1; j < srcSize; j++) { FacetLabel cp = srcTR.GetPath(j); int destOrdinal = destTr.GetOrdinal(cp); Assert.True(destOrdinal > 0, cp + " not found in destination"); Assert.AreEqual(destOrdinal, map[j]); } } } }
public virtual void TestOpenIfChangedReuse() { // test the reuse of data from the old DTR instance foreach (bool nrt in new bool[] { false, true }) { Directory dir = NewDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); FacetLabel cp_b = new FacetLabel("b"); writer.AddCategory(cp_b); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.IsNotNull(r2); // add r2's categories to the caches Assert.AreEqual(2, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(2)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.IsNull(r1.GetPath(2)); r1.Dispose(); r2.Dispose(); writer.Dispose(); dir.Dispose(); } }
public virtual void TestOpenIfChangedReuseAfterRecreate() { // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy Directory dir = NewDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); writer.Dispose(); DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); // now recreate, add a different category writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE); FacetLabel cp_b = new FacetLabel("b"); writer.AddCategory(cp_b); writer.Dispose(); DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.IsNotNull(r2); // fill r2's caches Assert.AreEqual(1, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(1)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.AreEqual(cp_a, r1.GetPath(1)); // check that r2 doesn't see cp_a Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a)); Assert.AreEqual(cp_b, r2.GetPath(1)); r2.Dispose(); r1.Dispose(); dir.Dispose(); }
public virtual void TestConcurrency() { // tests that addTaxonomy and addCategory work in parallel int numCategories = AtLeast(10000); // build an input taxonomy index Directory src = NewDirectory(); var tw = new DirectoryTaxonomyWriter(src); for (int i = 0; i < numCategories; i++) { tw.AddCategory(new FacetLabel("a", Convert.ToString(i))); } tw.Dispose(); // now add the taxonomy to an empty taxonomy, while adding the categories // again, in parallel -- in the end, no duplicate categories should exist. Directory dest = NewDirectory(); var destTw = new DirectoryTaxonomyWriter(dest); var t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw); t.Start(); IOrdinalMap map = new MemoryOrdinalMap(); destTw.AddTaxonomy(src, map); t.Join(); destTw.Dispose(); // now validate var dtr = new DirectoryTaxonomyReader(dest); // +2 to account for the root category + "a" Assert.AreEqual(numCategories + 2, dtr.Count); var categories = new JCG.HashSet <FacetLabel>(); for (int i = 1; i < dtr.Count; i++) { FacetLabel cat = dtr.GetPath(i); Assert.True(categories.Add(cat), "category " + cat + " already existed"); } dtr.Dispose(); IOUtils.Dispose(src, dest); }
public virtual void TestGetChildren() { Directory dir = NewDirectory(); var taxoWriter = new DirectoryTaxonomyWriter(dir); int numCategories = AtLeast(10); int numA = 0, numB = 0; Random random = Random; // add the two categories for which we'll also add children (so asserts are simpler) taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.AddCategory(new FacetLabel("b")); for (int i = 0; i < numCategories; i++) { if (random.NextBoolean()) { taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i, CultureInfo.InvariantCulture))); ++numA; } else { taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i, CultureInfo.InvariantCulture))); ++numB; } } // add category with no children taxoWriter.AddCategory(new FacetLabel("c")); taxoWriter.Dispose(); var taxoReader = new DirectoryTaxonomyReader(dir); // non existing category TaxonomyReader.ChildrenEnumerator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid"))); Assert.AreEqual(false, it.MoveNext()); // a category with no children it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c"))); Assert.AreEqual(false, it.MoveNext()); // arbitrary negative ordinal it = taxoReader.GetChildren(-2); Assert.AreEqual(false, it.MoveNext()); // root's children var roots = new JCG.HashSet <string> { "a", "b", "c" }; it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL); while (roots.Count > 0) { it.MoveNext(); FacetLabel root = taxoReader.GetPath(it.Current); Assert.AreEqual(1, root.Length); Assert.IsTrue(roots.Remove(root.Components[0])); } Assert.AreEqual(false, it.MoveNext()); for (int i = 0; i < 2; i++) { FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b"); int ordinal = taxoReader.GetOrdinal(cp); it = taxoReader.GetChildren(ordinal); int numChildren = 0; int child; while (it.MoveNext()) { child = it.Current; FacetLabel path = taxoReader.GetPath(child); Assert.AreEqual(2, path.Length); Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b"); ++numChildren; } int expected = i == 0 ? numA : numB; Assert.AreEqual(expected, numChildren, "invalid num children"); } taxoReader.Dispose(); dir.Dispose(); }
public virtual void TestConcurrency() { // tests that addTaxonomy and addCategory work in parallel int numCategories = AtLeast(10000); // build an input taxonomy index Directory src = NewDirectory(); var tw = new DirectoryTaxonomyWriter(src); for (int i = 0; i < numCategories; i++) { tw.AddCategory(new FacetLabel("a", Convert.ToString(i))); } tw.Dispose(); // now add the taxonomy to an empty taxonomy, while adding the categories // again, in parallel -- in the end, no duplicate categories should exist. Directory dest = NewDirectory(); var destTw = new DirectoryTaxonomyWriter(dest); ThreadClass t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw); t.Start(); OrdinalMap map = new MemoryOrdinalMap(); destTw.AddTaxonomy(src, map); t.Join(); destTw.Dispose(); // now validate var dtr = new DirectoryTaxonomyReader(dest); // +2 to account for the root category + "a" Assert.AreEqual(numCategories + 2, dtr.Size); var categories = new HashSet<FacetLabel>(); for (int i = 1; i < dtr.Size; i++) { FacetLabel cat = dtr.GetPath(i); Assert.True(categories.Add(cat), "category " + cat + " already existed"); } dtr.Dispose(); IOUtils.Close(src, dest); }
private void validate(Directory dest, Directory src, OrdinalMap ordMap) { var destTr = new DirectoryTaxonomyReader(dest); try { int destSize = destTr.Size; var srcTR = new DirectoryTaxonomyReader(src); try { var map = ordMap.Map; // validate taxo sizes int srcSize = srcTR.Size; Assert.True(destSize >= srcSize, "destination taxonomy expected to be larger than source; dest=" + destSize + " src=" + srcSize); // validate that all source categories exist in destination, and their // ordinals are as expected. for (int j = 1; j < srcSize; j++) { FacetLabel cp = srcTR.GetPath(j); int destOrdinal = destTr.GetOrdinal(cp); Assert.True(destOrdinal > 0, cp + " not found in destination"); Assert.AreEqual(destOrdinal, map[j]); } } finally { ((TaxonomyReader)srcTR).Dispose(true); } } finally { ((TaxonomyReader)destTr).Dispose(true); } }
public virtual void TestOpenIfChangedReuseAfterRecreate() { // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy Directory dir = NewDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); writer.Dispose(); DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); // now recreate, add a different category writer = new DirectoryTaxonomyWriter(dir, IndexWriterConfig.OpenMode_e.CREATE); FacetLabel cp_b = new FacetLabel("b"); writer.AddCategory(cp_b); writer.Dispose(); DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.NotNull(r2); // fill r2's caches Assert.AreEqual(1, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(1)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.AreEqual(cp_a, r1.GetPath(1)); // check that r2 doesn't see cp_a Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a)); Assert.AreEqual(cp_b, r2.GetPath(1)); (r2).Dispose(); (r1).Dispose(); dir.Dispose(); }
public virtual void TestGetChildren() { Directory dir = NewDirectory(); var taxoWriter = new DirectoryTaxonomyWriter(dir); int numCategories = AtLeast(10); int numA = 0, numB = 0; Random random = Random(); // add the two categories for which we'll also add children (so asserts are simpler) taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.AddCategory(new FacetLabel("b")); for (int i = 0; i < numCategories; i++) { if (random.NextBoolean()) { taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i))); ++numA; } else { taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i))); ++numB; } } // add category with no children taxoWriter.AddCategory(new FacetLabel("c")); taxoWriter.Dispose(); var taxoReader = new DirectoryTaxonomyReader(dir); // non existing category TaxonomyReader.ChildrenIterator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid"))); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); // a category with no children it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c"))); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); // arbitrary negative ordinal it = taxoReader.GetChildren(-2); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); // root's children var roots = new HashSet<string>(Arrays.AsList("a", "b", "c")); it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL); while (roots.Count > 0) { FacetLabel root = taxoReader.GetPath(it.Next()); Assert.AreEqual(1, root.Length); Assert.True(roots.Remove(root.Components[0])); } Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); for (int i = 0; i < 2; i++) { FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b"); int ordinal = taxoReader.GetOrdinal(cp); it = taxoReader.GetChildren(ordinal); int numChildren = 0; int child; while ((child = it.Next()) != TaxonomyReader.INVALID_ORDINAL) { FacetLabel path = taxoReader.GetPath(child); Assert.AreEqual(2, path.Length); Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b"); ++numChildren; } int expected = i == 0 ? numA : numB; Assert.AreEqual(expected, numChildren, "invalid num children"); } taxoReader.Dispose(); dir.Dispose(); }