public override void Run() { Random random = Random(); while (numCats.DecrementAndGet() > 0) { try { int value = random.Next(range); FacetLabel cp = new FacetLabel(Convert.ToString(value / 1000), Convert.ToString(value / 10000), Convert.ToString(value / 100000), Convert.ToString(value)); int ord = tw.AddCategory(cp); Assert.True(tw.GetParent(ord) != -1, "invalid parent for ordinal " + ord + ", category " + cp); string l1 = FacetsConfig.PathToString(cp.Components, 1); string l2 = FacetsConfig.PathToString(cp.Components, 2); string l3 = FacetsConfig.PathToString(cp.Components, 3); string l4 = FacetsConfig.PathToString(cp.Components, 4); values[l1] = l1; values[l2] = l2; values[l3] = l3; values[l4] = l4; } catch (IOException e) { throw new Exception(e.Message, e); } } }
public override void Run() { for (int i = 0; i < numCategories; i++) { try { destTW.AddCategory(new FacetLabel("a", Convert.ToString(i))); } catch (IOException e) { // shouldn't happen - if it does, let the test fail on uncaught exception. throw new Exception(e.ToString(), e); } } }
public virtual void TestWriterTwice3() { var indexDir = NewDirectory(); // First, create and fill the taxonomy var tw = new DirectoryTaxonomyWriter(indexDir); FillTaxonomy(tw); tw.Dispose(); // Now, open the same taxonomy and add the same categories again. // After a few categories, the LuceneTaxonomyWriter implementation // will stop looking for each category on disk, and rather read them // all into memory and close it's reader. The bug was that it closed // the reader, but forgot that it did (because it didn't set the reader // reference to null). tw = new DirectoryTaxonomyWriter(indexDir); FillTaxonomy(tw); // Add one new category, just to make commit() do something: tw.AddCategory(new FacetLabel("hi")); // Do a commit(). Here was a bug - if tw had a reader open, it should // be reopened after the commit. However, in our case the reader should // not be open (as explained above) but because it was not set to null, // we forgot that, tried to reopen it, and got an AlreadyClosedException. tw.Commit(); Assert.AreEqual(ExpectedCategories.Length + 1, tw.Size); tw.Dispose(); indexDir.Dispose(); }
public virtual void TestWriterSimpler() { var indexDir = NewDirectory(); var tw = new DirectoryTaxonomyWriter(indexDir); Assert.AreEqual(1, tw.Size); // the root only // Test that adding a new top-level category works Assert.AreEqual(1, tw.AddCategory(new FacetLabel("a"))); Assert.AreEqual(2, tw.Size); // Test that adding the same category again is noticed, and the // same ordinal (and not a new one) is returned. Assert.AreEqual(1, tw.AddCategory(new FacetLabel("a"))); Assert.AreEqual(2, tw.Size); // Test that adding another top-level category returns a new ordinal, // not the same one Assert.AreEqual(2, tw.AddCategory(new FacetLabel("b"))); Assert.AreEqual(3, tw.Size); // Test that adding a category inside one of the above adds just one // new ordinal: Assert.AreEqual(3, tw.AddCategory(new FacetLabel("a", "c"))); Assert.AreEqual(4, tw.Size); // Test that adding the same second-level category doesn't do anything: Assert.AreEqual(3, tw.AddCategory(new FacetLabel("a", "c"))); Assert.AreEqual(4, tw.Size); // Test that adding a second-level category with two new components // indeed adds two categories Assert.AreEqual(5, tw.AddCategory(new FacetLabel("d", "e"))); Assert.AreEqual(6, tw.Size); // Verify that the parents were added above in the order we expected Assert.AreEqual(4, tw.AddCategory(new FacetLabel("d"))); // Similar, but inside a category that already exists: Assert.AreEqual(7, tw.AddCategory(new FacetLabel("b", "d", "e"))); Assert.AreEqual(8, tw.Size); // And now inside two levels of categories that already exist: Assert.AreEqual(8, tw.AddCategory(new FacetLabel("b", "d", "f"))); Assert.AreEqual(9, tw.Size); tw.Dispose(); indexDir.Dispose(); }
public virtual void TestWriterLock() { // native fslock impl gets angry if we use it, so use RAMDirectory explicitly. var indexDir = new RAMDirectory(); var tw = new DirectoryTaxonomyWriter(indexDir); tw.AddCategory(new FacetLabel("hi", "there")); tw.Commit(); // we deliberately not close the write now, and keep it open and // locked. // Verify that the writer worked: var tr = new DirectoryTaxonomyReader(indexDir); Assert.AreEqual(2, tr.GetOrdinal(new FacetLabel("hi", "there"))); // Try to open a second writer, with the first one locking the directory. // We expect to get a LockObtainFailedException. try { Assert.Null(new DirectoryTaxonomyWriter(indexDir)); Fail("should have failed to write in locked directory"); } catch (LockObtainFailedException) { // this is what we expect to happen. } // Remove the lock, and now the open should succeed, and we can // write to the new writer. DirectoryTaxonomyWriter.Unlock(indexDir); var tw2 = new DirectoryTaxonomyWriter(indexDir); tw2.AddCategory(new FacetLabel("hey")); tw2.Dispose(); // See that the writer indeed wrote: var newtr = TaxonomyReader.OpenIfChanged(tr); Assert.NotNull(newtr); tr.Dispose(); tr = newtr; Assert.AreEqual(3, tr.GetOrdinal(new FacetLabel("hey"))); tr.Dispose(); tw.Dispose(); indexDir.Dispose(); }
public virtual void TestTaxonomyReaderRefreshRaces() { // compute base child arrays - after first chunk, and after the other var indexDirBase = NewDirectory(); var twBase = new DirectoryTaxonomyWriter(indexDirBase); twBase.AddCategory(new FacetLabel("a", "0")); FacetLabel abPath = new FacetLabel("a", "b"); twBase.AddCategory(abPath); twBase.Commit(); var trBase = new DirectoryTaxonomyReader(indexDirBase); ParallelTaxonomyArrays ca1 = trBase.ParallelTaxonomyArrays; int abOrd = trBase.GetOrdinal(abPath); int abYoungChildBase1 = ca1.Children()[abOrd]; int numCategories = AtLeast(800); for (int i = 0; i < numCategories; i++) { twBase.AddCategory(new FacetLabel("a", "b", Convert.ToString(i))); } twBase.Dispose(); var newTaxoReader = TaxonomyReader.OpenIfChanged(trBase); Assert.NotNull(newTaxoReader); trBase.Dispose(); trBase = newTaxoReader; ParallelTaxonomyArrays ca2 = trBase.ParallelTaxonomyArrays; int abYoungChildBase2 = ca2.Children()[abOrd]; int numRetries = AtLeast(50); for (int retry = 0; retry < numRetries; retry++) { AssertConsistentYoungestChild(abPath, abOrd, abYoungChildBase1, abYoungChildBase2, retry, numCategories); } trBase.Dispose(); indexDirBase.Dispose(); }
public virtual void TestSeparateReaderAndWriter2() { var indexDir = NewDirectory(); var tw = new DirectoryTaxonomyWriter(indexDir); tw.Commit(); var tr = new DirectoryTaxonomyReader(indexDir); // Test getOrdinal(): FacetLabel author = new FacetLabel("Author"); Assert.AreEqual(1, tr.Size); // the empty taxonomy has size 1 (the root) Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(author)); tw.AddCategory(author); // before commit and refresh, no change: Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(author)); Assert.AreEqual(1, tr.Size); // still root only... Assert.Null(TaxonomyReader.OpenIfChanged(tr)); // this is not enough, because tw.Commit() hasn't been done yet Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(author)); Assert.AreEqual(1, tr.Size); // still root only... tw.Commit(); // still not enough before refresh: Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(author)); Assert.AreEqual(1, tr.Size); // still root only... var newTaxoReader = TaxonomyReader.OpenIfChanged(tr); Assert.NotNull(newTaxoReader); tr.Dispose(); tr = newTaxoReader; Assert.AreEqual(1, tr.GetOrdinal(author)); Assert.AreEqual(2, tr.Size); tw.Dispose(); tr.Dispose(); indexDir.Dispose(); }
public virtual void TestSeparateReaderAndWriter() { var indexDir = NewDirectory(); var tw = new DirectoryTaxonomyWriter(indexDir); tw.Commit(); var tr = new DirectoryTaxonomyReader(indexDir); Assert.AreEqual(1, tr.Size); // the empty taxonomy has size 1 (the root) tw.AddCategory(new FacetLabel("Author")); Assert.AreEqual(1, tr.Size); // still root only... Assert.Null(TaxonomyReader.OpenIfChanged(tr)); // this is not enough, because tw.Commit() hasn't been done yet Assert.AreEqual(1, tr.Size); // still root only... tw.Commit(); Assert.AreEqual(1, tr.Size); // still root only... var newTaxoReader = TaxonomyReader.OpenIfChanged(tr); Assert.NotNull(newTaxoReader); tr.Dispose(); tr = newTaxoReader; int author = 1; try { Assert.AreEqual(TaxonomyReader.ROOT_ORDINAL, tr.ParallelTaxonomyArrays.Parents()[author]); // ok } catch (System.IndexOutOfRangeException) { Fail("After category addition, commit() and refresh(), getParent for " + author + " should NOT throw exception"); } Assert.AreEqual(2, tr.Size); // finally, see there are two categories // now, add another category, and verify that after commit and refresh // the parent of this category is correct (this requires the reader // to correctly update its prefetched parent vector), and that the // old information also wasn't ruined: tw.AddCategory(new FacetLabel("Author", "Richard Dawkins")); int dawkins = 2; tw.Commit(); newTaxoReader = TaxonomyReader.OpenIfChanged(tr); Assert.NotNull(newTaxoReader); tr.Dispose(); tr = newTaxoReader; int[] parents = tr.ParallelTaxonomyArrays.Parents(); Assert.AreEqual(author, parents[dawkins]); Assert.AreEqual(TaxonomyReader.ROOT_ORDINAL, parents[author]); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, parents[TaxonomyReader.ROOT_ORDINAL]); Assert.AreEqual(3, tr.Size); tw.Dispose(); tr.Dispose(); indexDir.Dispose(); }
public virtual void TestNrt() { var dir = NewDirectory(); var writer = new DirectoryTaxonomyWriter(dir); var reader = new DirectoryTaxonomyReader(writer); FacetLabel cp = new FacetLabel("a"); writer.AddCategory(cp); var newReader = TaxonomyReader.OpenIfChanged(reader); Assert.NotNull(newReader, "expected a new instance"); Assert.AreEqual(2, newReader.Size); Assert.AreNotSame(TaxonomyReader.INVALID_ORDINAL, newReader.GetOrdinal(cp)); reader.Dispose(); reader = newReader; writer.Dispose(); reader.Dispose(); dir.Dispose(); }
public virtual void TestChildrenArraysGrowth() { var indexDir = NewDirectory(); var tw = new DirectoryTaxonomyWriter(indexDir); tw.AddCategory(new FacetLabel("hi", "there")); tw.Commit(); var tr = new DirectoryTaxonomyReader(indexDir); ParallelTaxonomyArrays ca = tr.ParallelTaxonomyArrays; Assert.AreEqual(3, tr.Size); Assert.AreEqual(3, ca.Siblings().Length); Assert.AreEqual(3, ca.Children().Length); Assert.True(Arrays.Equals(new int[] { 1, 2, -1 }, ca.Children())); Assert.True(Arrays.Equals(new int[] { -1, -1, -1 }, ca.Siblings())); tw.AddCategory(new FacetLabel("hi", "ho")); tw.AddCategory(new FacetLabel("hello")); tw.Commit(); // Before refresh, nothing changed.. ParallelTaxonomyArrays newca = tr.ParallelTaxonomyArrays; Assert.AreSame(newca, ca); // we got exactly the same object Assert.AreEqual(3, tr.Size); Assert.AreEqual(3, ca.Siblings().Length); Assert.AreEqual(3, ca.Children().Length); // After the refresh, things change: var newtr = TaxonomyReader.OpenIfChanged(tr); Assert.NotNull(newtr); tr.Dispose(); tr = newtr; ca = tr.ParallelTaxonomyArrays; Assert.AreEqual(5, tr.Size); Assert.AreEqual(5, ca.Siblings().Length); Assert.AreEqual(5, ca.Children().Length); Assert.True(Arrays.Equals(new int[] { 4, 3, -1, -1, -1 }, ca.Children())); Assert.True(Arrays.Equals(new int[] { -1, -1, -1, 2, 1 }, ca.Siblings())); tw.Dispose(); tr.Dispose(); indexDir.Dispose(); }
private void AssertConsistentYoungestChild(FacetLabel abPath, int abOrd, int abYoungChildBase1, int abYoungChildBase2, int retry, int numCategories) { var indexDir = new SlowRAMDirectory(-1, null); // no slowness for intialization var tw = new DirectoryTaxonomyWriter(indexDir); tw.AddCategory(new FacetLabel("a", "0")); tw.AddCategory(abPath); tw.Commit(); var tr = new DirectoryTaxonomyReader(indexDir); for (int i = 0; i < numCategories; i++) { var cp = new FacetLabel("a", "b", Convert.ToString(i)); tw.AddCategory(cp); Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(cp), "Ordinal of " + cp + " must be invalid until Taxonomy Reader was refreshed"); } tw.Dispose(); var stop = new AtomicBoolean(false); Exception[] error = new Exception[] { null }; int[] retrieval = new int[] { 0 }; var thread = new ThreadAnonymousInnerClassHelper(this, abPath, abOrd, abYoungChildBase1, abYoungChildBase2, retry, tr, stop, error, retrieval); thread.Start(); indexDir.SleepMillis = 1; // some delay for refresh var newTaxoReader = TaxonomyReader.OpenIfChanged(tr); if (newTaxoReader != null) { newTaxoReader.Dispose(); } stop.Set(true); thread.Join(); Assert.Null(error[0], "Unexpcted exception at retry " + retry + " retrieval " + retrieval[0] + ": \n" + stackTraceStr(error[0])); tr.Dispose(); }
public virtual void TestReplaceTaxonomyDirectory() { Store.Directory indexDir = NewDirectory(); Store.Directory taxoDir = NewDirectory(); IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); var tw = new DirectoryTaxonomyWriter(taxoDir); w.Commit(); tw.Commit(); Store.Directory taxoDir2 = NewDirectory(); var tw2 = new DirectoryTaxonomyWriter(taxoDir2); tw2.AddCategory(new FacetLabel("a", "b")); tw2.Dispose(); var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null); SearcherAndTaxonomy pair = mgr.Acquire(); try { Assert.AreEqual(1, pair.taxonomyReader.Size); } finally { mgr.Release(pair); } w.AddDocument(new Document()); tw.ReplaceTaxonomy(taxoDir2); taxoDir2.Dispose(); w.Commit(); tw.Commit(); mgr.MaybeRefresh(); pair = mgr.Acquire(); try { Assert.AreEqual(3, pair.taxonomyReader.Size); } finally { mgr.Release(pair); } IOUtils.Close(mgr, tw, w, taxoDir, indexDir); }