public virtual void TestRecreateAndRefresh() { // DirTaxoWriter lost the INDEX_EPOCH property if it was opened in // CREATE_OR_APPEND (or commit(userData) called twice), which could lead to // DirTaxoReader succeeding to refresh(). Directory dir = NewDirectory(); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); TouchTaxo(taxoWriter, new FacetLabel("a")); var taxoReader = new DirectoryTaxonomyReader(dir); TouchTaxo(taxoWriter, new FacetLabel("b")); var newtr = TaxonomyReader.OpenIfChanged(taxoReader); taxoReader.Dispose(); taxoReader = newtr; Assert.AreEqual(1, Convert.ToInt32(taxoReader.CommitUserData[DirectoryTaxonomyWriter.INDEX_EPOCH])); // now recreate the taxonomy, and check that the epoch is preserved after opening DirTW again. taxoWriter.Dispose(); taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE); TouchTaxo(taxoWriter, new FacetLabel("c")); taxoWriter.Dispose(); taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); TouchTaxo(taxoWriter, new FacetLabel("d")); taxoWriter.Dispose(); newtr = TaxonomyReader.OpenIfChanged(taxoReader); taxoReader.Dispose(); taxoReader = newtr; Assert.AreEqual(2, Convert.ToInt32(taxoReader.CommitUserData[DirectoryTaxonomyWriter.INDEX_EPOCH])); taxoReader.Dispose(); dir.Dispose(); }
public virtual void TestConcurrency() { // tests that addTaxonomy and addCategory work in parallel int numCategories = AtLeast(10000); // build an input taxonomy index Directory src = NewDirectory(); var tw = new DirectoryTaxonomyWriter(src); for (int i = 0; i < numCategories; i++) { tw.AddCategory(new FacetLabel("a", Convert.ToString(i))); } tw.Dispose(); // now add the taxonomy to an empty taxonomy, while adding the categories // again, in parallel -- in the end, no duplicate categories should exist. Directory dest = NewDirectory(); var destTw = new DirectoryTaxonomyWriter(dest); var t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw); t.Start(); IOrdinalMap map = new MemoryOrdinalMap(); destTw.AddTaxonomy(src, map); t.Join(); destTw.Dispose(); // now validate var dtr = new DirectoryTaxonomyReader(dest); // +2 to account for the root category + "a" Assert.AreEqual(numCategories + 2, dtr.Count); var categories = new JCG.HashSet <FacetLabel>(); for (int i = 1; i < dtr.Count; i++) { FacetLabel cat = dtr.GetPath(i); Assert.True(categories.Add(cat), "category " + cat + " already existed"); } dtr.Dispose(); IOUtils.Dispose(src, dest); }
public virtual void TestCloseAfterIncRef() { Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Dispose(); DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir); ltr.IncRef(); ltr.Dispose(); // should not fail as we IncRef() before close var tmpSie = ltr.Size; ltr.DecRef(); dir.Dispose(); }
public virtual void TestOpenIfChangedReuse() { // test the reuse of data from the old DTR instance foreach (bool nrt in new bool[] { false, true }) { Directory dir = NewDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); FacetLabel cp_b = new FacetLabel("b"); writer.AddCategory(cp_b); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.IsNotNull(r2); // add r2's categories to the caches Assert.AreEqual(2, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(2)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.IsNull(r1.GetPath(2)); r1.Dispose(); r2.Dispose(); writer.Dispose(); dir.Dispose(); } }
public virtual void TestOpenIfChangedReuseAfterRecreate() { // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy Directory dir = NewDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); writer.Dispose(); DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); // now recreate, add a different category writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE); FacetLabel cp_b = new FacetLabel("b"); writer.AddCategory(cp_b); writer.Dispose(); DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.IsNotNull(r2); // fill r2's caches Assert.AreEqual(1, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(1)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.AreEqual(cp_a, r1.GetPath(1)); // check that r2 doesn't see cp_a Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a)); Assert.AreEqual(cp_b, r2.GetPath(1)); r2.Dispose(); r1.Dispose(); dir.Dispose(); }
public virtual void TestCloseAfterIncRef() { Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Dispose(); DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir); ltr.IncRef(); ltr.Dispose(); // should not fail as we IncRef() before close var _ = ltr.Count; ltr.DecRef(); dir.Dispose(); }
public virtual void TestAlreadyClosed() { Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Dispose(); var ltr = new DirectoryTaxonomyReader(dir); ltr.Dispose(); try { var tmpSize = ltr.Size; Fail("An AlreadyClosedException should have been thrown here"); } catch (AlreadyClosedException) { // good! } dir.Dispose(); }
public virtual void TestBackwardsCompatibility() { // tests that if the taxonomy index doesn't have the INDEX_EPOCH // property (supports pre-3.6 indexes), all still works. Directory dir = NewDirectory(); // create an empty index first, so that DirTaxoWriter initializes indexEpoch to 1. new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).Dispose(); var taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); taxoWriter.Dispose(); var taxoReader = new DirectoryTaxonomyReader(dir); Assert.AreEqual(1, Convert.ToInt32(taxoReader.CommitUserData[DirectoryTaxonomyWriter.INDEX_EPOCH], CultureInfo.InvariantCulture)); Assert.IsNull(TaxonomyReader.OpenIfChanged(taxoReader)); taxoReader.Dispose(); dir.Dispose(); }
public virtual void TestAlreadyClosed() { Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Dispose(); var ltr = new DirectoryTaxonomyReader(dir); ltr.Dispose(); try { var _ = ltr.Count; fail("An ObjectDisposedException should have been thrown here"); } catch (ObjectDisposedException) { // good! } dir.Dispose(); }
public virtual void TestGetChildren() { Directory dir = NewDirectory(); var taxoWriter = new DirectoryTaxonomyWriter(dir); int numCategories = AtLeast(10); int numA = 0, numB = 0; Random random = Random; // add the two categories for which we'll also add children (so asserts are simpler) taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.AddCategory(new FacetLabel("b")); for (int i = 0; i < numCategories; i++) { if (random.NextBoolean()) { taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i, CultureInfo.InvariantCulture))); ++numA; } else { taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i, CultureInfo.InvariantCulture))); ++numB; } } // add category with no children taxoWriter.AddCategory(new FacetLabel("c")); taxoWriter.Dispose(); var taxoReader = new DirectoryTaxonomyReader(dir); // non existing category TaxonomyReader.ChildrenEnumerator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid"))); Assert.AreEqual(false, it.MoveNext()); // a category with no children it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c"))); Assert.AreEqual(false, it.MoveNext()); // arbitrary negative ordinal it = taxoReader.GetChildren(-2); Assert.AreEqual(false, it.MoveNext()); // root's children var roots = new JCG.HashSet <string> { "a", "b", "c" }; it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL); while (roots.Count > 0) { it.MoveNext(); FacetLabel root = taxoReader.GetPath(it.Current); Assert.AreEqual(1, root.Length); Assert.IsTrue(roots.Remove(root.Components[0])); } Assert.AreEqual(false, it.MoveNext()); for (int i = 0; i < 2; i++) { FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b"); int ordinal = taxoReader.GetOrdinal(cp); it = taxoReader.GetChildren(ordinal); int numChildren = 0; int child; while (it.MoveNext()) { child = it.Current; FacetLabel path = taxoReader.GetPath(child); Assert.AreEqual(2, path.Length); Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b"); ++numChildren; } int expected = i == 0 ? numA : numB; Assert.AreEqual(expected, numChildren, "invalid num children"); } taxoReader.Dispose(); dir.Dispose(); }
public virtual void TestGetChildren() { Directory dir = NewDirectory(); var taxoWriter = new DirectoryTaxonomyWriter(dir); int numCategories = AtLeast(10); int numA = 0, numB = 0; Random random = Random(); // add the two categories for which we'll also add children (so asserts are simpler) taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.AddCategory(new FacetLabel("b")); for (int i = 0; i < numCategories; i++) { if (random.NextBoolean()) { taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i))); ++numA; } else { taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i))); ++numB; } } // add category with no children taxoWriter.AddCategory(new FacetLabel("c")); taxoWriter.Dispose(); var taxoReader = new DirectoryTaxonomyReader(dir); // non existing category TaxonomyReader.ChildrenIterator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid"))); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); // a category with no children it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c"))); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); // arbitrary negative ordinal it = taxoReader.GetChildren(-2); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); // root's children var roots = new HashSet<string>(Arrays.AsList("a", "b", "c")); it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL); while (roots.Count > 0) { FacetLabel root = taxoReader.GetPath(it.Next()); Assert.AreEqual(1, root.Length); Assert.True(roots.Remove(root.Components[0])); } Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); for (int i = 0; i < 2; i++) { FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b"); int ordinal = taxoReader.GetOrdinal(cp); it = taxoReader.GetChildren(ordinal); int numChildren = 0; int child; while ((child = it.Next()) != TaxonomyReader.INVALID_ORDINAL) { FacetLabel path = taxoReader.GetPath(child); Assert.AreEqual(2, path.Length); Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b"); ++numChildren; } int expected = i == 0 ? numA : numB; Assert.AreEqual(expected, numChildren, "invalid num children"); } taxoReader.Dispose(); dir.Dispose(); }
public virtual void TestOpenIfChangedAndRefCount() { Directory dir = new RAMDirectory(); // no need for random directories here var taxoWriter = new DirectoryTaxonomyWriter(dir); taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.Commit(); var taxoReader = new DirectoryTaxonomyReader(dir); Assert.AreEqual(1, taxoReader.RefCount, "wrong refCount"); taxoReader.IncRef(); Assert.AreEqual(2, taxoReader.RefCount, "wrong refCount"); taxoWriter.AddCategory(new FacetLabel("a", "b")); taxoWriter.Commit(); var newtr = TaxonomyReader.OpenIfChanged(taxoReader); Assert.NotNull(newtr); taxoReader.Dispose(); taxoReader = newtr; Assert.AreEqual(1, taxoReader.RefCount, "wrong refCount"); taxoWriter.Dispose(); taxoReader.Dispose(); dir.Dispose(); }
public virtual void TestOpenIfChangedManySegments() { // test openIfChanged() when the taxonomy contains many segments Directory dir = NewDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriterAnonymousInnerClassHelper(this, dir); var reader = new DirectoryTaxonomyReader(writer); int numRounds = Random().Next(10) + 10; int numCategories = 1; // one for root for (int i = 0; i < numRounds; i++) { int numCats = Random().Next(4) + 1; for (int j = 0; j < numCats; j++) { writer.AddCategory(new FacetLabel(Convert.ToString(i), Convert.ToString(j))); } numCategories += numCats + 1; // one for round-parent var newtr = TaxonomyReader.OpenIfChanged(reader); Assert.NotNull(newtr); reader.Dispose(); reader = newtr; // assert categories Assert.AreEqual(numCategories, reader.Size); int roundOrdinal = reader.GetOrdinal(new FacetLabel(Convert.ToString(i))); int[] parents = reader.ParallelTaxonomyArrays.Parents(); Assert.AreEqual(0, parents[roundOrdinal]); // round's parent is root for (int j = 0; j < numCats; j++) { int ord = reader.GetOrdinal(new FacetLabel(Convert.ToString(i), Convert.ToString(j))); Assert.AreEqual(roundOrdinal, parents[ord]); // round's parent is root } } reader.Dispose(); writer.Dispose(); dir.Dispose(); }
public virtual void TestConcurrency() { // tests that addTaxonomy and addCategory work in parallel int numCategories = AtLeast(10000); // build an input taxonomy index Directory src = NewDirectory(); var tw = new DirectoryTaxonomyWriter(src); for (int i = 0; i < numCategories; i++) { tw.AddCategory(new FacetLabel("a", Convert.ToString(i))); } tw.Dispose(); // now add the taxonomy to an empty taxonomy, while adding the categories // again, in parallel -- in the end, no duplicate categories should exist. Directory dest = NewDirectory(); var destTw = new DirectoryTaxonomyWriter(dest); ThreadClass t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw); t.Start(); OrdinalMap map = new MemoryOrdinalMap(); destTw.AddTaxonomy(src, map); t.Join(); destTw.Dispose(); // now validate var dtr = new DirectoryTaxonomyReader(dest); // +2 to account for the root category + "a" Assert.AreEqual(numCategories + 2, dtr.Size); var categories = new HashSet<FacetLabel>(); for (int i = 1; i < dtr.Size; i++) { FacetLabel cat = dtr.GetPath(i); Assert.True(categories.Add(cat), "category " + cat + " already existed"); } dtr.Dispose(); IOUtils.Close(src, dest); }
public virtual void TestOpenIfChangedMergedSegment() { // test openIfChanged() when all index segments were merged - used to be // a bug in ParentArray, caught by testOpenIfChangedManySegments - only // this test is not random Directory dir = NewDirectory(); // hold onto IW to forceMerge // note how we don't close it, since DTW will close it. IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(new LogByteSizeMergePolicy())); var writer = new DirectoryTaxonomyWriterAnonymousInnerClassHelper2(this, dir, iw) as DirectoryTaxonomyWriter; var reader = new DirectoryTaxonomyReader(writer); Assert.AreEqual(1, reader.Size); Assert.AreEqual(1, reader.ParallelTaxonomyArrays.Parents().Length); // add category and call forceMerge -- this should flush IW and merge segments down to 1 // in ParentArray.initFromReader, this used to fail assuming there are no parents. writer.AddCategory(new FacetLabel("1")); iw.ForceMerge(1); // now calling openIfChanged should trip on the bug var newtr = TaxonomyReader.OpenIfChanged(reader); Assert.NotNull(newtr); reader.Dispose(); reader = newtr; Assert.AreEqual(2, reader.Size); Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length); reader.Dispose(); writer.Dispose(); dir.Dispose(); }
public virtual void TestRecreateAndRefresh() { // DirTaxoWriter lost the INDEX_EPOCH property if it was opened in // CREATE_OR_APPEND (or commit(userData) called twice), which could lead to // DirTaxoReader succeeding to refresh(). Directory dir = NewDirectory(); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); TouchTaxo(taxoWriter, new FacetLabel("a")); var taxoReader = new DirectoryTaxonomyReader(dir); TouchTaxo(taxoWriter, new FacetLabel("b")); var newtr = TaxonomyReader.OpenIfChanged(taxoReader); taxoReader.Dispose(); taxoReader = newtr; Assert.AreEqual(1, Convert.ToInt32(taxoReader.CommitUserData[DirectoryTaxonomyWriter.INDEX_EPOCH])); // now recreate the taxonomy, and check that the epoch is preserved after opening DirTW again. taxoWriter.Dispose(); taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE); TouchTaxo(taxoWriter, new FacetLabel("c")); taxoWriter.Dispose(); taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); TouchTaxo(taxoWriter, new FacetLabel("d")); taxoWriter.Dispose(); newtr = TaxonomyReader.OpenIfChanged(taxoReader); taxoReader.Dispose(); taxoReader = newtr; Assert.AreEqual(2, Convert.ToInt32(taxoReader.CommitUserData[DirectoryTaxonomyWriter.INDEX_EPOCH])); taxoReader.Dispose(); dir.Dispose(); }
public virtual void TestOpenIfChangedNoChangesButSegmentMerges() { // test openIfChanged() when the taxonomy hasn't really changed, but segments // were merged. The NRT reader will be reopened, and ParentArray used to assert // that the new reader contains more ordinals than were given from the old // TaxReader version Directory dir = NewDirectory(); // hold onto IW to forceMerge // note how we don't close it, since DTW will close it. var iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(new LogByteSizeMergePolicy())); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriterAnonymousInnerClassHelper3(this, dir, iw); // add a category so that the following DTR open will cause a flush and // a new segment will be created writer.AddCategory(new FacetLabel("a")); var reader = new DirectoryTaxonomyReader(writer); Assert.AreEqual(2, reader.Size); Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length); // merge all the segments so that NRT reader thinks there's a change iw.ForceMerge(1); // now calling openIfChanged should trip on the wrong assert in ParetArray's ctor var newtr = TaxonomyReader.OpenIfChanged(reader); Assert.NotNull(newtr); reader.Dispose(); reader = newtr; Assert.AreEqual(2, reader.Size); Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length); reader.Dispose(); writer.Dispose(); dir.Dispose(); }
private void doTestReadRecreatedTaxonomy(Random random, bool closeReader) { Directory dir = null; TaxonomyWriter tw = null; TaxonomyReader tr = null; // prepare a few categories int n = 10; FacetLabel[] cp = new FacetLabel[n]; for (int i = 0; i < n; i++) { cp[i] = new FacetLabel("a", Convert.ToString(i)); } try { dir = NewDirectory(); tw = new DirectoryTaxonomyWriter(dir); tw.AddCategory(new FacetLabel("a")); tw.Dispose(); tr = new DirectoryTaxonomyReader(dir); int baseNumCategories = tr.Size; for (int i = 0; i < n; i++) { int k = random.Next(n); tw = new DirectoryTaxonomyWriter(dir, IndexWriterConfig.OpenMode_e.CREATE); for (int j = 0; j <= k; j++) { tw.AddCategory(cp[j]); } tw.Dispose(); if (closeReader) { tr.Dispose(true); tr = new DirectoryTaxonomyReader(dir); } else { var newtr = TaxonomyReader.OpenIfChanged(tr); Assert.NotNull(newtr); tr.Dispose(true); tr = newtr; } Assert.AreEqual(baseNumCategories + 1 + k, tr.Size, "Wrong #categories in taxonomy (i=" + i + ", k=" + k + ")"); } } finally { IOUtils.Close(tr as DirectoryTaxonomyReader, tw, dir); } }