public AddCategory ( FacetLabel categoryPath ) : int | ||
categoryPath | FacetLabel | |
Результат | int |
private void TouchTaxo(DirectoryTaxonomyWriter taxoWriter, FacetLabel cp) { taxoWriter.AddCategory(cp); taxoWriter.SetCommitData(new Dictionary <string, string>() { { "just", "data" } }); taxoWriter.Commit(); }
public virtual void TestOpenIfChangedReuse() { // test the reuse of data from the old DTR instance foreach (bool nrt in new bool[] { false, true }) { Directory dir = NewDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); FacetLabel cp_b = new FacetLabel("b"); writer.AddCategory(cp_b); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.IsNotNull(r2); // add r2's categories to the caches Assert.AreEqual(2, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(2)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.IsNull(r1.GetPath(2)); r1.Dispose(); r2.Dispose(); writer.Dispose(); dir.Dispose(); } }
public virtual void TestCommitUserData() { // Verifies taxonomy commit data Directory dir = NewDirectory(); var taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.AddCategory(new FacetLabel("b")); IDictionary <string, string> userCommitData = new Dictionary <string, string>(); userCommitData["testing"] = "1 2 3"; taxoWriter.SetCommitData(userCommitData); taxoWriter.Dispose(); var r = DirectoryReader.Open(dir); Assert.AreEqual(3, r.NumDocs, "2 categories plus root should have been committed to the underlying directory"); var readUserCommitData = r.IndexCommit.UserData; Assert.IsTrue("1 2 3".Equals(readUserCommitData["testing"], StringComparison.Ordinal), "wrong value extracted from commit data"); Assert.IsNotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in commitData", readUserCommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]); r.Dispose(); // open DirTaxoWriter again and commit, INDEX_EPOCH should still exist // in the commit data, otherwise DirTaxoReader.refresh() might not detect // that the taxonomy index has been recreated. taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); taxoWriter.AddCategory(new FacetLabel("c")); // add a category so that commit will happen taxoWriter.SetCommitData(new Dictionary <string, string>() { { "just", "data" } }); taxoWriter.Commit(); // verify taxoWriter.getCommitData() Assert.IsTrue(taxoWriter.CommitData.ContainsKey(DirectoryTaxonomyWriter.INDEX_EPOCH), DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in taoxWriter.commitData"); taxoWriter.Dispose(); r = DirectoryReader.Open(dir); readUserCommitData = r.IndexCommit.UserData; Assert.IsTrue(readUserCommitData.ContainsKey(DirectoryTaxonomyWriter.INDEX_EPOCH), DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in taoxWriter.commitData"); r.Dispose(); dir.Dispose(); }
public virtual void TestAddToEmpty() { Directory dest = NewDirectory(); Directory src = NewDirectory(); DirectoryTaxonomyWriter srcTW = new DirectoryTaxonomyWriter(src); srcTW.AddCategory(new FacetLabel("Author", "Rob Pike")); srcTW.AddCategory(new FacetLabel("Aardvarks", "Bob")); srcTW.Dispose(); DirectoryTaxonomyWriter destTW = new DirectoryTaxonomyWriter(dest); OrdinalMap map = randomOrdinalMap(); destTW.AddTaxonomy(src, map); destTW.Dispose(); validate(dest, src, map); IOUtils.Close(dest, src); }
private void doTestReadRecreatedTaxonomy(Random random, bool closeReader) { Directory dir = null; ITaxonomyWriter tw = null; TaxonomyReader tr = null; // prepare a few categories int n = 10; FacetLabel[] cp = new FacetLabel[n]; for (int i = 0; i < n; i++) { cp[i] = new FacetLabel("a", Convert.ToString(i, CultureInfo.InvariantCulture)); } try { dir = NewDirectory(); tw = new DirectoryTaxonomyWriter(dir); tw.AddCategory(new FacetLabel("a")); tw.Dispose(); tr = new DirectoryTaxonomyReader(dir); int baseNumCategories = tr.Count; for (int i = 0; i < n; i++) { int k = random.Next(n); tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE); for (int j = 0; j <= k; j++) { tw.AddCategory(cp[j]); } tw.Dispose(); if (closeReader) { tr.Dispose(); tr = new DirectoryTaxonomyReader(dir); } else { var newtr = TaxonomyReader.OpenIfChanged(tr); Assert.IsNotNull(newtr); tr.Dispose(); tr = newtr; } Assert.AreEqual(baseNumCategories + 1 + k, tr.Count, "Wrong #categories in taxonomy (i=" + i + ", k=" + k + ")"); } } finally { IOUtils.Dispose(tr, tw, dir); } }
public virtual void TestRollback() { // Verifies that if rollback is called, DTW is closed. Directory dir = NewDirectory(); var dtw = new DirectoryTaxonomyWriter(dir); dtw.AddCategory(new FacetLabel("a")); dtw.Rollback(); try { dtw.AddCategory(new FacetLabel("a")); Fail("should not have succeeded to add a category following rollback."); } catch (AlreadyClosedException) { // expected } dir.Dispose(); }
public virtual void TestReplaceTaxonomy() { Directory input = NewDirectory(); var taxoWriter = new DirectoryTaxonomyWriter(input); int ordA = taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.Dispose(); Directory dir = NewDirectory(); taxoWriter = new DirectoryTaxonomyWriter(dir); int ordB = taxoWriter.AddCategory(new FacetLabel("b")); taxoWriter.AddCategory(new FacetLabel("c")); taxoWriter.Commit(); long origEpoch = getEpoch(dir); // replace the taxonomy with the input one taxoWriter.ReplaceTaxonomy(input); // LUCENE-4633: make sure that category "a" is not added again in any case taxoWriter.AddTaxonomy(input, new MemoryOrdinalMap()); Assert.AreEqual(2, taxoWriter.Size, "no categories should have been added"); // root + 'a' Assert.AreEqual(ordA, taxoWriter.AddCategory(new FacetLabel("a")), "category 'a' received new ordinal?"); // add the same category again -- it should not receive the same ordinal ! int newOrdB = taxoWriter.AddCategory(new FacetLabel("b")); Assert.AreNotSame(ordB, newOrdB, "new ordinal cannot be the original ordinal"); Assert.AreEqual(2, newOrdB, "ordinal should have been 2 since only one category was added by replaceTaxonomy"); taxoWriter.Dispose(); long newEpoch = getEpoch(dir); Assert.True(origEpoch < newEpoch, "index epoch should have been updated after replaceTaxonomy"); dir.Dispose(); input.Dispose(); }
public virtual void TestAddToEmpty() { Directory dest = NewDirectory(); Directory src = NewDirectory(); DirectoryTaxonomyWriter srcTW = new DirectoryTaxonomyWriter(src); srcTW.AddCategory(new FacetLabel("Author", "Rob Pike")); srcTW.AddCategory(new FacetLabel("Aardvarks", "Bob")); srcTW.Dispose(); DirectoryTaxonomyWriter destTW = new DirectoryTaxonomyWriter(dest); IOrdinalMap map = randomOrdinalMap(); destTW.AddTaxonomy(src, map); destTW.Dispose(); validate(dest, src, map); IOUtils.Dispose(dest, src); }
public override void Run() { for (int i = 0; i < numCategories; i++) { try { destTW.AddCategory(new FacetLabel("a", Convert.ToString(i, CultureInfo.InvariantCulture))); } catch (IOException e) { // shouldn't happen - if it does, let the test fail on uncaught exception. throw new Exception(e.ToString(), e); } } }
public virtual void TestCloseTwice() { Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Dispose(); var ltr = new DirectoryTaxonomyReader(dir); ltr.Dispose(); ltr.Dispose(); // no exception should be thrown dir.Dispose(); }
public virtual void TestConcurrency() { // tests that addTaxonomy and addCategory work in parallel int numCategories = AtLeast(10000); // build an input taxonomy index Directory src = NewDirectory(); var tw = new DirectoryTaxonomyWriter(src); for (int i = 0; i < numCategories; i++) { tw.AddCategory(new FacetLabel("a", Convert.ToString(i))); } tw.Dispose(); // now add the taxonomy to an empty taxonomy, while adding the categories // again, in parallel -- in the end, no duplicate categories should exist. Directory dest = NewDirectory(); var destTw = new DirectoryTaxonomyWriter(dest); var t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw); t.Start(); IOrdinalMap map = new MemoryOrdinalMap(); destTw.AddTaxonomy(src, map); t.Join(); destTw.Dispose(); // now validate var dtr = new DirectoryTaxonomyReader(dest); // +2 to account for the root category + "a" Assert.AreEqual(numCategories + 2, dtr.Count); var categories = new JCG.HashSet <FacetLabel>(); for (int i = 1; i < dtr.Count; i++) { FacetLabel cat = dtr.GetPath(i); Assert.True(categories.Add(cat), "category " + cat + " already existed"); } dtr.Dispose(); IOUtils.Dispose(src, dest); }
public virtual void TestCloseAfterIncRef() { Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Dispose(); DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir); ltr.IncRef(); ltr.Dispose(); // should not fail as we IncRef() before close var tmpSie = ltr.Size; ltr.DecRef(); dir.Dispose(); }
public override void Run() { Random random = Random; while (numCats.DecrementAndGet() > 0) { string cat = Convert.ToString(random.Next(range)); try { tw.AddCategory(new FacetLabel("a", cat)); } catch (IOException e) { throw new Exception(e.ToString(), e); } } }
public override void Run() { Random random = Random; while (numCats.DecrementAndGet() > 0) { string cat = Convert.ToString(random.Next(range), CultureInfo.InvariantCulture); try { tw.AddCategory(new FacetLabel("a", cat)); } catch (Exception e) when(e.IsIOException()) { throw RuntimeException.Create(e); } } }
public virtual void TestEnsureOpen() { // verifies that an exception is thrown if DTW was closed Directory dir = NewDirectory(); DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir); dtw.Dispose(); try { dtw.AddCategory(new FacetLabel("a")); Fail("should not have succeeded to add a category following close."); } catch (AlreadyClosedException) { // expected } dir.Dispose(); }
public virtual void TestCommit() { // Verifies that nothing is committed to the underlying Directory, if // commit() wasn't called. Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); Assert.False(DirectoryReader.IndexExists(dir)); ltw.Commit(); // first commit, so that an index will be created ltw.AddCategory(new FacetLabel("a")); IndexReader r = DirectoryReader.Open(dir); Assert.AreEqual(1, r.NumDocs, "No categories should have been committed to the underlying directory"); r.Dispose(); ltw.Dispose(); dir.Dispose(); }
public virtual void TestOpenIfChangedReuseAfterRecreate() { // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy Directory dir = NewDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); writer.Dispose(); DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); // now recreate, add a different category writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE); FacetLabel cp_b = new FacetLabel("b"); writer.AddCategory(cp_b); writer.Dispose(); DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.IsNotNull(r2); // fill r2's caches Assert.AreEqual(1, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(1)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.AreEqual(cp_a, r1.GetPath(1)); // check that r2 doesn't see cp_a Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a)); Assert.AreEqual(cp_b, r2.GetPath(1)); r2.Dispose(); r1.Dispose(); dir.Dispose(); }
public virtual void TestCloseNoEmptyCommits() { // LUCENE-4972: DTW used to create empty commits even if no changes were made Directory dir = NewDirectory(); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir); taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.Commit(); long gen1 = SegmentInfos.GetLastCommitGeneration(dir); taxoWriter.Dispose(); long gen2 = SegmentInfos.GetLastCommitGeneration(dir); Assert.AreEqual(gen1, gen2, "empty commit should not have changed the index"); taxoWriter.Dispose(); dir.Dispose(); }
public virtual void TestAlreadyClosed() { Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Dispose(); var ltr = new DirectoryTaxonomyReader(dir); ltr.Dispose(); try { var tmpSize = ltr.Size; Fail("An AlreadyClosedException should have been thrown here"); } catch (AlreadyClosedException) { // good! } dir.Dispose(); }
public virtual void TestCloseAfterIncRef() { Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Dispose(); DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir); ltr.IncRef(); ltr.Dispose(); // should not fail as we IncRef() before close var _ = ltr.Count; ltr.DecRef(); dir.Dispose(); }
public virtual void TestAlreadyClosed() { Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Dispose(); var ltr = new DirectoryTaxonomyReader(dir); ltr.Dispose(); try { var _ = ltr.Count; fail("An ObjectDisposedException should have been thrown here"); } catch (ObjectDisposedException) { // good! } dir.Dispose(); }
public virtual void TestReplaceTaxoWithLargeTaxonomy() { var srcTaxoDir = NewDirectory(); var targetTaxoDir = NewDirectory(); // build source, large, taxonomy DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(srcTaxoDir); int ord = taxoWriter.AddCategory(new FacetLabel("A", "1", "1", "1", "1", "1", "1")); taxoWriter.Dispose(); taxoWriter = new DirectoryTaxonomyWriter(targetTaxoDir); int ordinal = taxoWriter.AddCategory(new FacetLabel("B", "1")); Assert.AreEqual(1, taxoWriter.GetParent(ordinal)); // call getParent to initialize taxoArrays taxoWriter.Commit(); taxoWriter.ReplaceTaxonomy(srcTaxoDir); Assert.AreEqual(ord - 1, taxoWriter.GetParent(ord)); taxoWriter.Dispose(); srcTaxoDir.Dispose(); targetTaxoDir.Dispose(); }
public virtual void TestOpenIfChangedResult() { Directory dir = null; DirectoryTaxonomyWriter ltw = null; DirectoryTaxonomyReader ltr = null; try { dir = NewDirectory(); ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Commit(); ltr = new DirectoryTaxonomyReader(dir); Assert.Null(TaxonomyReader.OpenIfChanged(ltr), "Nothing has changed"); ltw.AddCategory(new FacetLabel("b")); ltw.Commit(); DirectoryTaxonomyReader newtr = TaxonomyReader.OpenIfChanged(ltr); Assert.NotNull(newtr, "changes were committed"); Assert.Null(TaxonomyReader.OpenIfChanged(newtr), "Nothing has changed"); (newtr).Dispose(); } finally { IOUtils.Close(ltw, ltr, dir); } }
public virtual void TestSimple() { Directory dest = NewDirectory(); var tw1 = new DirectoryTaxonomyWriter(dest); tw1.AddCategory(new FacetLabel("Author", "Mark Twain")); tw1.AddCategory(new FacetLabel("Animals", "Dog")); tw1.AddCategory(new FacetLabel("Author", "Rob Pike")); Directory src = NewDirectory(); var tw2 = new DirectoryTaxonomyWriter(src); tw2.AddCategory(new FacetLabel("Author", "Rob Pike")); tw2.AddCategory(new FacetLabel("Aardvarks", "Bob")); tw2.Dispose(); OrdinalMap map = randomOrdinalMap(); tw1.AddTaxonomy(src, map); tw1.Dispose(); validate(dest, src, map); IOUtils.Close(dest, src); }
public virtual void TestOpenIfChangedReuse() { // test the reuse of data from the old DTR instance foreach (bool nrt in new bool[] { false, true }) { Directory dir = NewDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); FacetLabel cp_b = new FacetLabel("b"); writer.AddCategory(cp_b); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.NotNull(r2); // add r2's categories to the caches Assert.AreEqual(2, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(2)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.Null(r1.GetPath(2)); (r1).Dispose(); (r2).Dispose(); writer.Dispose(); dir.Dispose(); } }
public virtual void TestOpenIfChangedReuseAfterRecreate() { // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy Directory dir = NewDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); writer.Dispose(); DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); // now recreate, add a different category writer = new DirectoryTaxonomyWriter(dir, IndexWriterConfig.OpenMode_e.CREATE); FacetLabel cp_b = new FacetLabel("b"); writer.AddCategory(cp_b); writer.Dispose(); DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.NotNull(r2); // fill r2's caches Assert.AreEqual(1, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(1)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.AreEqual(cp_a, r1.GetPath(1)); // check that r2 doesn't see cp_a Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a)); Assert.AreEqual(cp_b, r2.GetPath(1)); (r2).Dispose(); (r1).Dispose(); dir.Dispose(); }
public virtual void TestCloseTwice() { Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir); ltw.AddCategory(new FacetLabel("a")); ltw.Dispose(); var ltr = new DirectoryTaxonomyReader(dir); (ltr).Dispose(); (ltr).Dispose(); // no exception should be thrown dir.Dispose(); }
public virtual void TestOpenIfChangedReplaceTaxonomy() { // test openIfChanged when replaceTaxonomy is called, which is equivalent to recreate // only can work with NRT as well Directory src = NewDirectory(); DirectoryTaxonomyWriter w = new DirectoryTaxonomyWriter(src); FacetLabel cp_b = new FacetLabel("b"); w.AddCategory(cp_b); w.Dispose(); foreach (bool nrt in new bool[] { false, true }) { Directory dir = NewDirectory(); var writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.AddCategory(cp_a); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir); // fill r1's caches Assert.AreEqual(1, r1.GetOrdinal(cp_a)); Assert.AreEqual(cp_a, r1.GetPath(1)); // now replace taxonomy writer.ReplaceTaxonomy(src); if (!nrt) { writer.Commit(); } DirectoryTaxonomyReader r2 = TaxonomyReader.OpenIfChanged(r1); Assert.NotNull(r2); // fill r2's caches Assert.AreEqual(1, r2.GetOrdinal(cp_b)); Assert.AreEqual(cp_b, r2.GetPath(1)); // check that r1 doesn't see cp_b Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r1.GetOrdinal(cp_b)); Assert.AreEqual(cp_a, r1.GetPath(1)); // check that r2 doesn't see cp_a Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, r2.GetOrdinal(cp_a)); Assert.AreEqual(cp_b, r2.GetPath(1)); (r2).Dispose(); (r1).Dispose(); writer.Dispose(); dir.Dispose(); } src.Dispose(); }
public virtual void TestReplaceTaxoWithLargeTaxonomy() { var srcTaxoDir = NewDirectory(); var targetTaxoDir = NewDirectory(); // build source, large, taxonomy DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(srcTaxoDir); int ord = taxoWriter.AddCategory(new FacetLabel("A", "1", "1", "1", "1", "1", "1")); taxoWriter.Dispose(); taxoWriter = new DirectoryTaxonomyWriter(targetTaxoDir); int ordinal = taxoWriter.AddCategory(new FacetLabel("B", "1")); Assert.AreEqual(1, taxoWriter.GetParent(ordinal)); // call getParent to initialize taxoArrays taxoWriter.Commit(); taxoWriter.ReplaceTaxonomy(srcTaxoDir); Assert.AreEqual(ord - 1, taxoWriter.GetParent(ord)); taxoWriter.Dispose(); srcTaxoDir.Dispose(); targetTaxoDir.Dispose(); }
public virtual void TestOpenIfChangedAndRefCount() { Directory dir = new RAMDirectory(); // no need for random directories here var taxoWriter = new DirectoryTaxonomyWriter(dir); taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.Commit(); var taxoReader = new DirectoryTaxonomyReader(dir); Assert.AreEqual(1, taxoReader.RefCount, "wrong refCount"); taxoReader.IncRef(); Assert.AreEqual(2, taxoReader.RefCount, "wrong refCount"); taxoWriter.AddCategory(new FacetLabel("a", "b")); taxoWriter.Commit(); var newtr = TaxonomyReader.OpenIfChanged(taxoReader); Assert.NotNull(newtr); taxoReader.Dispose(); taxoReader = newtr; Assert.AreEqual(1, taxoReader.RefCount, "wrong refCount"); taxoWriter.Dispose(); taxoReader.Dispose(); dir.Dispose(); }
public virtual void TestGetChildren() { Directory dir = NewDirectory(); var taxoWriter = new DirectoryTaxonomyWriter(dir); int numCategories = AtLeast(10); int numA = 0, numB = 0; Random random = Random; // add the two categories for which we'll also add children (so asserts are simpler) taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.AddCategory(new FacetLabel("b")); for (int i = 0; i < numCategories; i++) { if (random.NextBoolean()) { taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i, CultureInfo.InvariantCulture))); ++numA; } else { taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i, CultureInfo.InvariantCulture))); ++numB; } } // add category with no children taxoWriter.AddCategory(new FacetLabel("c")); taxoWriter.Dispose(); var taxoReader = new DirectoryTaxonomyReader(dir); // non existing category TaxonomyReader.ChildrenEnumerator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid"))); Assert.AreEqual(false, it.MoveNext()); // a category with no children it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c"))); Assert.AreEqual(false, it.MoveNext()); // arbitrary negative ordinal it = taxoReader.GetChildren(-2); Assert.AreEqual(false, it.MoveNext()); // root's children var roots = new JCG.HashSet <string> { "a", "b", "c" }; it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL); while (roots.Count > 0) { it.MoveNext(); FacetLabel root = taxoReader.GetPath(it.Current); Assert.AreEqual(1, root.Length); Assert.IsTrue(roots.Remove(root.Components[0])); } Assert.AreEqual(false, it.MoveNext()); for (int i = 0; i < 2; i++) { FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b"); int ordinal = taxoReader.GetOrdinal(cp); it = taxoReader.GetChildren(ordinal); int numChildren = 0; int child; while (it.MoveNext()) { child = it.Current; FacetLabel path = taxoReader.GetPath(child); Assert.AreEqual(2, path.Length); Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b"); ++numChildren; } int expected = i == 0 ? numA : numB; Assert.AreEqual(expected, numChildren, "invalid num children"); } taxoReader.Dispose(); dir.Dispose(); }
private void doTestReadRecreatedTaxonomy(Random random, bool closeReader) { Directory dir = null; TaxonomyWriter tw = null; TaxonomyReader tr = null; // prepare a few categories int n = 10; FacetLabel[] cp = new FacetLabel[n]; for (int i = 0; i < n; i++) { cp[i] = new FacetLabel("a", Convert.ToString(i)); } try { dir = NewDirectory(); tw = new DirectoryTaxonomyWriter(dir); tw.AddCategory(new FacetLabel("a")); tw.Dispose(); tr = new DirectoryTaxonomyReader(dir); int baseNumCategories = tr.Size; for (int i = 0; i < n; i++) { int k = random.Next(n); tw = new DirectoryTaxonomyWriter(dir, IndexWriterConfig.OpenMode_e.CREATE); for (int j = 0; j <= k; j++) { tw.AddCategory(cp[j]); } tw.Dispose(); if (closeReader) { tr.Dispose(true); tr = new DirectoryTaxonomyReader(dir); } else { var newtr = TaxonomyReader.OpenIfChanged(tr); Assert.NotNull(newtr); tr.Dispose(true); tr = newtr; } Assert.AreEqual(baseNumCategories + 1 + k, tr.Size, "Wrong #categories in taxonomy (i=" + i + ", k=" + k + ")"); } } finally { IOUtils.Close(tr as DirectoryTaxonomyReader, tw, dir); } }
public virtual void TestReplaceTaxonomy() { Directory input = NewDirectory(); var taxoWriter = new DirectoryTaxonomyWriter(input); int ordA = taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.Dispose(); Directory dir = NewDirectory(); taxoWriter = new DirectoryTaxonomyWriter(dir); int ordB = taxoWriter.AddCategory(new FacetLabel("b")); taxoWriter.AddCategory(new FacetLabel("c")); taxoWriter.Commit(); long origEpoch = getEpoch(dir); // replace the taxonomy with the input one taxoWriter.ReplaceTaxonomy(input); // LUCENE-4633: make sure that category "a" is not added again in any case taxoWriter.AddTaxonomy(input, new MemoryOrdinalMap()); Assert.AreEqual(2, taxoWriter.Count, "no categories should have been added"); // root + 'a' Assert.AreEqual(ordA, taxoWriter.AddCategory(new FacetLabel("a")), "category 'a' received new ordinal?"); // add the same category again -- it should not receive the same ordinal ! int newOrdB = taxoWriter.AddCategory(new FacetLabel("b")); Assert.AreNotSame(ordB, newOrdB, "new ordinal cannot be the original ordinal"); Assert.AreEqual(2, newOrdB, "ordinal should have been 2 since only one category was added by replaceTaxonomy"); taxoWriter.Dispose(); long newEpoch = getEpoch(dir); Assert.True(origEpoch < newEpoch, "index epoch should have been updated after replaceTaxonomy"); dir.Dispose(); input.Dispose(); }
private void TouchTaxo(DirectoryTaxonomyWriter taxoWriter, FacetLabel cp) { taxoWriter.AddCategory(cp); taxoWriter.CommitData = new Dictionary<string, string>() { {"just", "data"} }; taxoWriter.Commit(); }
public virtual void TestConcurrency() { // tests that addTaxonomy and addCategory work in parallel int numCategories = AtLeast(10000); // build an input taxonomy index Directory src = NewDirectory(); var tw = new DirectoryTaxonomyWriter(src); for (int i = 0; i < numCategories; i++) { tw.AddCategory(new FacetLabel("a", Convert.ToString(i))); } tw.Dispose(); // now add the taxonomy to an empty taxonomy, while adding the categories // again, in parallel -- in the end, no duplicate categories should exist. Directory dest = NewDirectory(); var destTw = new DirectoryTaxonomyWriter(dest); ThreadClass t = new ThreadAnonymousInnerClassHelper2(this, numCategories, destTw); t.Start(); OrdinalMap map = new MemoryOrdinalMap(); destTw.AddTaxonomy(src, map); t.Join(); destTw.Dispose(); // now validate var dtr = new DirectoryTaxonomyReader(dest); // +2 to account for the root category + "a" Assert.AreEqual(numCategories + 2, dtr.Size); var categories = new HashSet<FacetLabel>(); for (int i = 1; i < dtr.Size; i++) { FacetLabel cat = dtr.GetPath(i); Assert.True(categories.Add(cat), "category " + cat + " already existed"); } dtr.Dispose(); IOUtils.Close(src, dest); }
public virtual void TestCommit() { // Verifies that nothing is committed to the underlying Directory, if // commit() wasn't called. Directory dir = NewDirectory(); var ltw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); Assert.False(DirectoryReader.IndexExists(dir)); ltw.Commit(); // first commit, so that an index will be created ltw.AddCategory(new FacetLabel("a")); IndexReader r = DirectoryReader.Open(dir); Assert.AreEqual(1, r.NumDocs, "No categories should have been committed to the underlying directory"); r.Dispose(); ltw.Dispose(); dir.Dispose(); }
public virtual void TestCommitUserData() { // Verifies taxonomy commit data Directory dir = NewDirectory(); var taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.AddCategory(new FacetLabel("b")); IDictionary<string, string> userCommitData = new Dictionary<string, string>(); userCommitData["testing"] = "1 2 3"; taxoWriter.CommitData = userCommitData; taxoWriter.Dispose(); var r = DirectoryReader.Open(dir); Assert.AreEqual(3, r.NumDocs, "2 categories plus root should have been committed to the underlying directory"); var readUserCommitData = r.IndexCommit.UserData; Assert.True("1 2 3".Equals(readUserCommitData["testing"]), "wrong value extracted from commit data"); Assert.NotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in commitData", readUserCommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]); r.Dispose(); // open DirTaxoWriter again and commit, INDEX_EPOCH should still exist // in the commit data, otherwise DirTaxoReader.refresh() might not detect // that the taxonomy index has been recreated. taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); taxoWriter.AddCategory(new FacetLabel("c")); // add a category so that commit will happen taxoWriter.CommitData = new Dictionary<string, string>() { {"just", "data"} }; taxoWriter.Commit(); // verify taxoWriter.getCommitData() Assert.NotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in taoxWriter.commitData", taxoWriter.CommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]); taxoWriter.Dispose(); r = DirectoryReader.Open(dir); readUserCommitData = r.IndexCommit.UserData; Assert.NotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in commitData", readUserCommitData[DirectoryTaxonomyWriter.INDEX_EPOCH]); r.Dispose(); dir.Dispose(); }
public virtual void TestReaderFreshness() { // ensures that the internal index reader is always kept fresh. Previously, // this simple scenario failed, if the cache just evicted the category that // is being added. Directory dir = NewDirectory(); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE); int o1 = taxoWriter.AddCategory(new FacetLabel("a")); int o2 = taxoWriter.AddCategory(new FacetLabel("a")); Assert.True(o1 == o2, "ordinal for same category that is added twice should be the same !"); taxoWriter.Dispose(); dir.Dispose(); }
public virtual void TestRollback() { // Verifies that if rollback is called, DTW is closed. Directory dir = NewDirectory(); var dtw = new DirectoryTaxonomyWriter(dir); dtw.AddCategory(new FacetLabel("a")); dtw.Rollback(); try { dtw.AddCategory(new FacetLabel("a")); Fail("should not have succeeded to add a category following rollback."); } catch (AlreadyClosedException) { // expected } dir.Dispose(); }
public virtual void TestPrepareCommitNoEmptyCommits() { // LUCENE-4972: DTW used to create empty commits even if no changes were made Directory dir = NewDirectory(); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir); taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.PrepareCommit(); taxoWriter.Commit(); long gen1 = SegmentInfos.GetLastCommitGeneration(dir); taxoWriter.PrepareCommit(); taxoWriter.Commit(); long gen2 = SegmentInfos.GetLastCommitGeneration(dir); Assert.AreEqual(gen1, gen2, "empty commit should not have changed the index"); taxoWriter.Dispose(); dir.Dispose(); }
public virtual void TestHugeLabel() { Directory indexDir = NewDirectory(), taxoDir = NewDirectory(); IndexWriter indexWriter = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, new Cl2oTaxonomyWriterCache(2, 1f, 1)); FacetsConfig config = new FacetsConfig(); // Add one huge label: string bigs = null; int ordinal = -1; int len = FacetLabel.MAX_CATEGORY_PATH_LENGTH - 4; // for the dimension and separator bigs = TestUtil.RandomSimpleString(Random(), len, len); FacetField ff = new FacetField("dim", bigs); FacetLabel cp = new FacetLabel("dim", bigs); ordinal = taxoWriter.AddCategory(cp); Document doc = new Document(); doc.Add(ff); indexWriter.AddDocument(config.Build(taxoWriter, doc)); // Add tiny ones to cause a re-hash for (int i = 0; i < 3; i++) { string s = TestUtil.RandomSimpleString(Random(), 1, 10); taxoWriter.AddCategory(new FacetLabel("dim", s)); doc = new Document(); doc.Add(new FacetField("dim", s)); indexWriter.AddDocument(config.Build(taxoWriter, doc)); } // when too large components were allowed to be added, this resulted in a new added category Assert.AreEqual(ordinal, taxoWriter.AddCategory(cp)); IOUtils.Close(indexWriter, taxoWriter); DirectoryReader indexReader = DirectoryReader.Open(indexDir); var taxoReader = new DirectoryTaxonomyReader(taxoDir); IndexSearcher searcher = new IndexSearcher(indexReader); DrillDownQuery ddq = new DrillDownQuery(new FacetsConfig()); ddq.Add("dim", bigs); Assert.AreEqual(1, searcher.Search(ddq, 10).TotalHits); IOUtils.Close(indexReader, taxoReader, indexDir, taxoDir); }
public virtual void TestEnsureOpen() { // verifies that an exception is thrown if DTW was closed Directory dir = NewDirectory(); DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir); dtw.Dispose(); try { dtw.AddCategory(new FacetLabel("a")); Fail("should not have succeeded to add a category following close."); } catch (AlreadyClosedException) { // expected } dir.Dispose(); }
public virtual void TestGetChildren() { Directory dir = NewDirectory(); var taxoWriter = new DirectoryTaxonomyWriter(dir); int numCategories = AtLeast(10); int numA = 0, numB = 0; Random random = Random(); // add the two categories for which we'll also add children (so asserts are simpler) taxoWriter.AddCategory(new FacetLabel("a")); taxoWriter.AddCategory(new FacetLabel("b")); for (int i = 0; i < numCategories; i++) { if (random.NextBoolean()) { taxoWriter.AddCategory(new FacetLabel("a", Convert.ToString(i))); ++numA; } else { taxoWriter.AddCategory(new FacetLabel("b", Convert.ToString(i))); ++numB; } } // add category with no children taxoWriter.AddCategory(new FacetLabel("c")); taxoWriter.Dispose(); var taxoReader = new DirectoryTaxonomyReader(dir); // non existing category TaxonomyReader.ChildrenIterator it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("invalid"))); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); // a category with no children it = taxoReader.GetChildren(taxoReader.GetOrdinal(new FacetLabel("c"))); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); // arbitrary negative ordinal it = taxoReader.GetChildren(-2); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); // root's children var roots = new HashSet<string>(Arrays.AsList("a", "b", "c")); it = taxoReader.GetChildren(TaxonomyReader.ROOT_ORDINAL); while (roots.Count > 0) { FacetLabel root = taxoReader.GetPath(it.Next()); Assert.AreEqual(1, root.Length); Assert.True(roots.Remove(root.Components[0])); } Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, it.Next()); for (int i = 0; i < 2; i++) { FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b"); int ordinal = taxoReader.GetOrdinal(cp); it = taxoReader.GetChildren(ordinal); int numChildren = 0; int child; while ((child = it.Next()) != TaxonomyReader.INVALID_ORDINAL) { FacetLabel path = taxoReader.GetPath(child); Assert.AreEqual(2, path.Length); Assert.AreEqual(path.Components[0], i == 0 ? "a" : "b"); ++numChildren; } int expected = i == 0 ? numA : numB; Assert.AreEqual(expected, numChildren, "invalid num children"); } taxoReader.Dispose(); dir.Dispose(); }