private void DisableFullTextIndex() { if (this._dataset is WebDemandDataset) { WebDemandDataset ds = (WebDemandDataset)this._dataset; if (ds.UnderlyingDataset is FullTextIndexedDataset) { this._dataset = ds.UnderlyingDataset; this.DisableFullTextIndex(); this._dataset = new WebDemandDataset(this._dataset); } } else if (this._dataset is FullTextIndexedDataset) { SparqlOptimiser.RemoveOptimiser(this._ftOptimiser); this._ftOptimiser = null; this._ftSearcher.Dispose(); this._ftSearcher = null; this._dataset = ((FullTextIndexedDataset)this._dataset).UnderlyingDataset; this._ftIndexer.Dispose(); this._ftIndexer = null; this._ftIndex.Dispose(); this._ftIndex = null; } this._processor = new LeviathanQueryProcessor(this._dataset); }
public void FullTextIndexMultiOccurrenceRemoval() { IFullTextIndexer indexer = null; try { indexer = new LuceneObjectsIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, new DefaultIndexSchema()); Graph g = new Graph(); INode example = g.CreateLiteralNode("This is an example node which we'll index multiple times"); for (int i = 0; i < 10; i++) { g.Assert(new Triple(g.CreateBlankNode(), g.CreateUriNode(UriFactory.Create("ex:predicate")), example)); } indexer.Index(g); LuceneSearchProvider searcher = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index); Assert.Equal(10, searcher.Match("example").Count()); for (int i = 9; i >= 0; i--) { indexer.Unindex(g.Triples.First()); indexer.Flush(); Assert.Equal(i, searcher.Match("example").Count()); } } finally { if (indexer != null) { indexer.Dispose(); } } }
private void RunTest(IFullTextIndexer indexer, String query, IEnumerable <INode> expected) { this.EnsureTestData(); indexer.Index(this._dataset); indexer.Dispose(); //Build the SPARQL Query and parse it SparqlParameterizedString queryString = new SparqlParameterizedString(query); queryString.Namespaces = this.GetQueryNamespaces(); SparqlQuery q = this._parser.ParseFromString(queryString); SparqlFormatter formatter = new SparqlFormatter(q.NamespaceMap); Console.WriteLine("Parsed Query:"); Console.WriteLine(formatter.Format(q)); Console.WriteLine("Expected Results:"); foreach (INode n in expected) { Console.WriteLine(n.ToString(formatter)); } Console.WriteLine(); LuceneSearchProvider provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index); try { q.AlgebraOptimisers = new IAlgebraOptimiser[] { new FullTextOptimiser(provider) }; LeviathanQueryProcessor processor = new LeviathanQueryProcessor(this._dataset); SparqlResultSet results = processor.ProcessQuery(q) as SparqlResultSet; if (results != null) { TestTools.ShowResults(results); foreach (INode n in expected) { Assert.IsTrue(results.Any(r => r.HasValue("match") && r["match"] != null && r["match"].Equals(n)), "Did not get expected ?match => " + formatter.Format(n)); } foreach (SparqlResult r in results) { Assert.IsTrue(r.HasValue("match") && r["match"] != null && expected.Contains(r["match"]), "Unexpected Match " + formatter.Format(r["match"])); } } else { Assert.Fail("Did not get a SPARQL Result Set as expected"); } } finally { provider.Dispose(); LuceneTestHarness.Index.Dispose(); } }
private void RunTest(IFullTextIndexer indexer, String query, int expectedResults, bool exact) { this.EnsureTestData(); indexer.Index(this._dataset); indexer.Dispose(); //Build the SPARQL Query and parse it SparqlParameterizedString queryString = new SparqlParameterizedString(query); queryString.Namespaces = this.GetQueryNamespaces(); SparqlQuery q = this._parser.ParseFromString(queryString); SparqlFormatter formatter = new SparqlFormatter(q.NamespaceMap); Console.WriteLine("Parsed Query:"); Console.WriteLine(formatter.Format(q)); LuceneSearchProvider provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index); FullTextPropertyFunctionFactory factory = new FullTextPropertyFunctionFactory(); try { PropertyFunctionFactory.AddFactory(factory); q.AlgebraOptimisers = new IAlgebraOptimiser[] { new FullTextOptimiser(provider) }; Options.AlgebraOptimisation = true; LeviathanQueryProcessor processor = new LeviathanQueryProcessor(this._dataset); SparqlResultSet results = processor.ProcessQuery(q) as SparqlResultSet; if (results != null) { TestTools.ShowResults(results); if (exact) { Assert.Equal(expectedResults, results.Count); } else { Assert.True(expectedResults >= results.Count, "Got more results that the expected maximum"); } } else { Assert.True(false, "Did not get a SPARQL Result Set as expected"); } } finally { PropertyFunctionFactory.RemoveFactory(factory); provider.Dispose(); LuceneTestHarness.Index.Dispose(); } }
/// <summary> /// Creates a new Full Text Indexed Dataset /// </summary> /// <param name="dataset">Dataset to wrap</param> /// <param name="indexer">Indexer to use</param> /// <param name="indexNow">Whether the dataset provided should be indexed now, set to false if indexer is linked to an existing index for this data</param> /// <remarks> /// If <paramref name="indexNow"/> is true then the provided dataset will be fully indexed when this constructor is called /// </remarks> public FullTextIndexedDataset(ISparqlDataset dataset, IFullTextIndexer indexer, bool indexNow) : base(dataset) { this._indexer = indexer; //Index Now if requested this._indexNow = indexNow; if (indexNow) { foreach (IGraph g in this.Graphs) { this._indexer.Index(g); } } }
public void FullTextIndexCreationLucenePredicates() { IFullTextIndexer indexer = null; try { indexer = new LucenePredicatesIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema); indexer.Index(this.GetTestData()); } finally { if (indexer != null) { indexer.Dispose(); } } }
public void FullTextIndexSearchLucenePredicatesWithLimitAndThreshold() { IFullTextIndexer indexer = null; IFullTextSearchProvider provider = null; try { indexer = new LucenePredicatesIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema); indexer.Index(this.GetTestData()); } finally { if (indexer != null) { indexer.Dispose(); } } try { provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index); NTriplesFormatter formatter = new NTriplesFormatter(); int i = 0; foreach (IFullTextSearchResult result in provider.Match("http", 1.0d, 5)) { i++; Console.WriteLine(result.Node.ToString(formatter) + " - Scores " + result.Score); Assert.IsTrue(result.Score >= 1.0d, "Score should be higher than desired threshold"); } Assert.IsTrue(i <= 5, "Should be a max of 5 results"); } finally { if (provider != null) { provider.Dispose(); } } }
private void EnableFullTextIndex() { if (this._dataset is FullTextIndexedDataset) { //Nothing to do } else if (this._dataset is WebDemandDataset) { WebDemandDataset ds = (WebDemandDataset)this._dataset; this._dataset = ds.UnderlyingDataset; this.EnableFullTextIndex(); this._dataset = new WebDemandDataset(this._dataset); } else { //Create and ensure index ready for use this._ftIndex = new RAMDirectory(); var writer = new IndexWriter(this._ftIndex, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), IndexWriter.MaxFieldLength.UNLIMITED); writer.Dispose(); //Create Indexer and wrap dataset this._ftIndexer = new LuceneObjectsIndexer(this._ftIndex, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), new DefaultIndexSchema()); if (this._dataset is WebDemandDataset) { //Web Demand needs to go around Full Text as we want to index on demand loaded content this._dataset = new WebDemandDataset(new FullTextIndexedDataset(((WebDemandDataset)this._dataset).UnderlyingDataset, this._ftIndexer, true)); } else { this._dataset = new FullTextIndexedDataset(this._dataset, this._ftIndexer, true); } //Create and Register Optimizer this._ftSearcher = new LuceneSearchProvider(Lucene.Net.Util.Version.LUCENE_29, this._ftIndex); this._ftOptimiser = new FullTextOptimiser(this._ftSearcher); SparqlOptimiser.AddOptimiser(this._ftOptimiser); } this._processor = new LeviathanQueryProcessor(this._dataset); }
public void FullTextIndexSearchLuceneSubjectsWithThreshold() { IFullTextIndexer indexer = null; IFullTextSearchProvider provider = null; try { indexer = new LuceneSubjectsIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema); indexer.Index(this.GetTestData()); } finally { if (indexer != null) { indexer.Dispose(); } } try { provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index); NTriplesFormatter formatter = new NTriplesFormatter(); foreach (IFullTextSearchResult result in provider.Match("http", 0.75d)) { Console.WriteLine(result.Node.ToString(formatter) + " - Scores " + result.Score); Assert.IsTrue(result.Score >= 0.75d, "Score should be higher than desired threshold"); } } finally { if (provider != null) { provider.Dispose(); } } }
public void FullTextIndexSearchLuceneObjects() { IFullTextIndexer indexer = null; IFullTextSearchProvider provider = null; try { indexer = new LuceneObjectsIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema); indexer.Index(this.GetTestData()); } finally { if (indexer != null) { indexer.Dispose(); } } try { provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index); NTriplesFormatter formatter = new NTriplesFormatter(); foreach (IFullTextSearchResult result in provider.Match("http")) { Console.WriteLine(result.Node.ToString(formatter) + " - Scores " + result.Score); } } finally { if (provider != null) { provider.Dispose(); } } }
/// <summary> /// Tries to load an object based on information from the Configuration Graph /// </summary> /// <param name="g">Configuration Graph</param> /// <param name="objNode">Object Node</param> /// <param name="targetType">Target Type</param> /// <param name="obj">Output Object</param> /// <returns></returns> public bool TryLoadObject(IGraph g, INode objNode, Type targetType, out object obj) { obj = null; INode index = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "index")); //INode indexer = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "indexer")); INode searcher = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "searcher")); INode analyzer = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "analyzer")); INode schema = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "schema")); INode version = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "version")); Object tempIndex, tempAnalyzer, tempSchema; int ver = 2900; //Always check for the version ver = ConfigurationLoader.GetConfigurationInt32(g, objNode, version, 2900); switch (targetType.FullName) { case DefaultIndexSchema: obj = new DefaultIndexSchema(); break; case FullTextOptimiser: //Need to get the Search Provider INode providerNode = ConfigurationLoader.GetConfigurationNode(g, objNode, searcher); if (providerNode == null) { throw new DotNetRdfConfigurationException("Unable to load the Full Text Optimiser specified by the Node '" + objNode.ToString() + "' as there was no value specified for the required dnr-ft:searcher property"); } Object tempSearcher = ConfigurationLoader.LoadObject(g, providerNode); if (tempSearcher is IFullTextSearchProvider) { obj = new FullTextOptimiser((IFullTextSearchProvider)tempSearcher); } else { throw new DotNetRdfConfigurationException("Unable to load the Full Text Optimiser specified by the Node '" + objNode.ToString() + "' as the value specified for the dnr-ft:searcher property pointed to an object which could not be loaded as a type that implements the required IFullTextSearchProvider interface"); } break; case LuceneObjectsIndexer: case LucenePredicatesIndexer: case LuceneSubjectsIndexer: case LuceneSearchProvider: //For any Lucene Indexer/Search Provider need to know the Index, Analyzer and Schema to be used //Then get the Index tempIndex = ConfigurationLoader.GetConfigurationNode(g, objNode, index); if (tempIndex == null) { throw new DotNetRdfConfigurationException("Unable to load the Lucene Indexer specified by the Node '" + objNode.ToString() + "' as there was no value specified for the required dnr-ft:index property"); } tempIndex = ConfigurationLoader.LoadObject(g, (INode)tempIndex); if (tempIndex is Directory) { //Next get the Analyzer (assume Standard if none specified) tempAnalyzer = ConfigurationLoader.GetConfigurationNode(g, objNode, analyzer); if (tempAnalyzer == null) { tempAnalyzer = new StandardAnalyzer(this.GetLuceneVersion(ver)); } else { tempAnalyzer = ConfigurationLoader.LoadObject(g, (INode)tempAnalyzer); } if (tempAnalyzer is Analyzer) { //Finally get the Schema (assume Default if none specified) tempSchema = ConfigurationLoader.GetConfigurationNode(g, objNode, schema); if (tempSchema == null) { tempSchema = new DefaultIndexSchema(); } else { tempSchema = ConfigurationLoader.LoadObject(g, (INode)tempSchema); } if (tempSchema is IFullTextIndexSchema) { //Now we can create the Object switch (targetType.FullName) { case LuceneObjectsIndexer: obj = new LuceneObjectsIndexer((Directory)tempIndex, (Analyzer)tempAnalyzer, (IFullTextIndexSchema)tempSchema); break; case LucenePredicatesIndexer: obj = new LucenePredicatesIndexer((Directory)tempIndex, (Analyzer)tempAnalyzer, (IFullTextIndexSchema)tempSchema); break; case LuceneSubjectsIndexer: obj = new LuceneSubjectsIndexer((Directory)tempIndex, (Analyzer)tempAnalyzer, (IFullTextIndexSchema)tempSchema); break; case LuceneSearchProvider: //Before the Search Provider has been loaded determine whether we need to carry out auto-indexing List <INode> sources = ConfigurationLoader.GetConfigurationData(g, objNode, g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "buildIndexFor"))).ToList(); if (sources.Count > 0) { //If there are sources to index ensure we have an indexer to index with INode indexerNode = ConfigurationLoader.GetConfigurationNode(g, objNode, g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "buildIndexWith"))); if (indexerNode == null) { throw new DotNetRdfConfigurationException("Unable to load the Lucene Search Provider specified by the Node '" + objNode.ToString() + "' as there were values specified for the dnr-ft:buildIndexFor property but no dnr-ft:buildIndexWith property was found"); } IFullTextIndexer indexer = ConfigurationLoader.LoadObject(g, indexerNode) as IFullTextIndexer; if (indexer == null) { throw new DotNetRdfConfigurationException("Unable to load the Lucene Search Provider specified by the Node '" + objNode.ToString() + "' as the value given for the dnr-ft:buildIndexWith property pointed to an Object which could not be loaded as a type that implements the required IFullTextIndexer interface"); } try { //For Each Source load it and Index it foreach (INode sourceNode in sources) { Object source = ConfigurationLoader.LoadObject(g, sourceNode); if (source is ISparqlDataset) { indexer.Index((ISparqlDataset)source); } else if (source is ITripleStore) { foreach (IGraph graph in ((ITripleStore)source).Graphs) { indexer.Index(graph); } } else if (source is IGraph) { indexer.Index((IGraph)source); } else { throw new DotNetRdfConfigurationException("Unable to load the Lucene Search Provider specified by the Node '" + objNode.ToString() + "' as a value given for the dnr-ft:buildIndexFor property ('" + sourceNode.ToString() + "') pointed to an Object which could not be loaded as a type that implements one of the required interfaces: IGraph, ITripleStore or ISparqlDataset"); } } } finally { indexer.Dispose(); } } //Then we actually load the Search Provider obj = new LuceneSearchProvider(this.GetLuceneVersion(ver), (Directory)tempIndex, (Analyzer)tempAnalyzer, (IFullTextIndexSchema)tempSchema); break; } } else { throw new DotNetRdfConfigurationException("Unable to load the Lucene Indexer specified by the Node '" + objNode.ToString() + "' as the value given for the dnr-ft:schema property pointed to an Object which could not be loaded as a type that implements the required IFullTextIndexSchema interface"); } } else { throw new DotNetRdfConfigurationException("Unable to load the Lucene Indexer specified by the Node '" + objNode.ToString() + "' as the value given for the dnr-ft:analyzer property pointed to an Object which could not be loaded as a type that derives from the required Lucene.Net.Analysis.Analyzer type"); } } else { throw new DotNetRdfConfigurationException("Unable to load the Lucene Indexer specified by the Node '" + objNode.ToString() + "' as the value given for the dnr-ft:index property pointed to an Object which could not be loaded as a type that derives from the required Lucene.Net.Store.Directory type"); } break; default: try { if (this._luceneAnalyzerType.IsAssignableFrom(targetType)) { if (targetType.GetConstructor(new Type[] { typeof(LucVersion) }) != null) { obj = Activator.CreateInstance(targetType, new Object[] { this.GetLuceneVersion(ver) }); } else { obj = Activator.CreateInstance(targetType); } } else if (this._luceneDirectoryType.IsAssignableFrom(targetType)) { String dir = ConfigurationLoader.GetConfigurationString(g, objNode, ConfigurationLoader.CreateConfigurationNode(g, ConfigurationLoader.PropertyFromFile)); if (dir != null) { try { obj = Activator.CreateInstance(targetType, new Object[] { dir }); } catch { MethodInfo method = targetType.GetMethod("Open", new Type[] { typeof(DirInfo) }); if (method != null) { obj = method.Invoke(null, new Object[] { new DirInfo(ConfigurationLoader.ResolvePath(dir)) }); } } } else { obj = Activator.CreateInstance(targetType); } //Ensure the Index if necessary if (obj != null) { if (ConfigurationLoader.GetConfigurationBoolean(g, objNode, g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "ensureIndex")), false)) { IndexWriter writer = new IndexWriter((Directory)obj, new StandardAnalyzer(this.GetLuceneVersion(ver))); writer.Close(); } } } } catch { //Since we know we don't allow loading of all analyzers and directories we allow for users to inject other object factories //which may know how to load those specific instances obj = null; } break; } return(obj != null); }
private void ProcessResourceStream(IResource resource, IFullTextIndexer indexer) { indexer.AddDocumentFragment(resource.ID, GetResourceText(resource)); }
public void ProcessResourceText(int resID, IFullTextIndexer indexer) { IResource resource = _pluginEnvironment.ResourceStore.LoadResource(resID); ProcessResourceStream(resource, indexer); }
public void FullTextIndexDestructionLucenePredicates() { IFullTextIndexer indexer = null; try { LuceneSearchProvider provider = null; int origCount; try { provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index); origCount = provider.Match("http").Count(); provider.Dispose(); } catch { origCount = 0; } finally { if (provider != null) { provider.Dispose(); } } Console.WriteLine("Prior to indexing search returns " + origCount + " result(s)"); indexer = new LucenePredicatesIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema); IGraph g = this.GetTestData(); indexer.Index(g); indexer.Dispose(); indexer = null; int currCount; try { provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index); currCount = provider.Match("http").Count(); provider.Dispose(); } catch { currCount = 0; } finally { if (provider != null) { provider.Dispose(); } } Console.WriteLine("After indexing search returns " + currCount + " result(s)"); indexer = new LucenePredicatesIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema); indexer.Unindex(g); indexer.Dispose(); indexer = null; try { provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index); currCount = provider.Match("http").Count(); Console.WriteLine("After unindexing search returns " + currCount + " result(s)"); } finally { if (provider != null) { provider.Dispose(); } } Assert.Equal(origCount, currCount); } finally { if (indexer != null) { indexer.Dispose(); } LuceneTestHarness.Index.Dispose(); } }
/// <summary> /// Creates a new Full Text Indexed Dataset /// </summary> /// <param name="dataset">Dataset to wrap</param> /// <param name="indexer">Indexer to use</param> /// <remarks> /// Does not do any indexing, assumes the provided dataset is already indexed. When using this constructor only changes to the dataset affect the index /// </remarks> public FullTextIndexedDataset(ISparqlDataset dataset, IFullTextIndexer indexer) : this(dataset, indexer, false) { }
private void RunTest(IFullTextIndexer indexer, String query, int expectedResults) { this.RunTest(indexer, query, expectedResults, true); }