private void DisableFullTextIndex()
{
if (this._dataset is WebDemandDataset)
{
WebDemandDataset ds = (WebDemandDataset)this._dataset;
if (ds.UnderlyingDataset is FullTextIndexedDataset)
{
this._dataset = ds.UnderlyingDataset;
this.DisableFullTextIndex();
this._dataset = new WebDemandDataset(this._dataset);
}
}
else if (this._dataset is FullTextIndexedDataset)
{
SparqlOptimiser.RemoveOptimiser(this._ftOptimiser);
this._ftOptimiser = null;
this._ftSearcher.Dispose();
this._ftSearcher = null;
this._dataset = ((FullTextIndexedDataset)this._dataset).UnderlyingDataset;
this._ftIndexer.Dispose();
this._ftIndexer = null;
this._ftIndex.Dispose();
this._ftIndex = null;
}
this._processor = new LeviathanQueryProcessor(this._dataset);
}
public void FullTextIndexMultiOccurrenceRemoval()
{
IFullTextIndexer indexer = null;
try
{
indexer = new LuceneObjectsIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, new DefaultIndexSchema());
Graph g = new Graph();
INode example = g.CreateLiteralNode("This is an example node which we'll index multiple times");
for (int i = 0; i < 10; i++)
{
g.Assert(new Triple(g.CreateBlankNode(), g.CreateUriNode(UriFactory.Create("ex:predicate")), example));
}
indexer.Index(g);
LuceneSearchProvider searcher = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index);
Assert.Equal(10, searcher.Match("example").Count());
for (int i = 9; i >= 0; i--)
{
indexer.Unindex(g.Triples.First());
indexer.Flush();
Assert.Equal(i, searcher.Match("example").Count());
}
}
finally
{
if (indexer != null)
{
indexer.Dispose();
}
}
}
private void RunTest(IFullTextIndexer indexer, String query, IEnumerable <INode> expected)
{
this.EnsureTestData();
indexer.Index(this._dataset);
indexer.Dispose();
//Build the SPARQL Query and parse it
SparqlParameterizedString queryString = new SparqlParameterizedString(query);
queryString.Namespaces = this.GetQueryNamespaces();
SparqlQuery q = this._parser.ParseFromString(queryString);
SparqlFormatter formatter = new SparqlFormatter(q.NamespaceMap);
Console.WriteLine("Parsed Query:");
Console.WriteLine(formatter.Format(q));
Console.WriteLine("Expected Results:");
foreach (INode n in expected)
{
Console.WriteLine(n.ToString(formatter));
}
Console.WriteLine();
LuceneSearchProvider provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index);
try
{
q.AlgebraOptimisers = new IAlgebraOptimiser[] { new FullTextOptimiser(provider) };
LeviathanQueryProcessor processor = new LeviathanQueryProcessor(this._dataset);
SparqlResultSet results = processor.ProcessQuery(q) as SparqlResultSet;
if (results != null)
{
TestTools.ShowResults(results);
foreach (INode n in expected)
{
Assert.IsTrue(results.Any(r => r.HasValue("match") && r["match"] != null && r["match"].Equals(n)), "Did not get expected ?match => " + formatter.Format(n));
}
foreach (SparqlResult r in results)
{
Assert.IsTrue(r.HasValue("match") && r["match"] != null && expected.Contains(r["match"]), "Unexpected Match " + formatter.Format(r["match"]));
}
}
else
{
Assert.Fail("Did not get a SPARQL Result Set as expected");
}
}
finally
{
provider.Dispose();
LuceneTestHarness.Index.Dispose();
}
}
private void RunTest(IFullTextIndexer indexer, String query, int expectedResults, bool exact)
{
this.EnsureTestData();
indexer.Index(this._dataset);
indexer.Dispose();
//Build the SPARQL Query and parse it
SparqlParameterizedString queryString = new SparqlParameterizedString(query);
queryString.Namespaces = this.GetQueryNamespaces();
SparqlQuery q = this._parser.ParseFromString(queryString);
SparqlFormatter formatter = new SparqlFormatter(q.NamespaceMap);
Console.WriteLine("Parsed Query:");
Console.WriteLine(formatter.Format(q));
LuceneSearchProvider provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index);
FullTextPropertyFunctionFactory factory = new FullTextPropertyFunctionFactory();
try
{
PropertyFunctionFactory.AddFactory(factory);
q.AlgebraOptimisers = new IAlgebraOptimiser[] { new FullTextOptimiser(provider) };
Options.AlgebraOptimisation = true;
LeviathanQueryProcessor processor = new LeviathanQueryProcessor(this._dataset);
SparqlResultSet results = processor.ProcessQuery(q) as SparqlResultSet;
if (results != null)
{
TestTools.ShowResults(results);
if (exact)
{
Assert.Equal(expectedResults, results.Count);
}
else
{
Assert.True(expectedResults >= results.Count, "Got more results that the expected maximum");
}
}
else
{
Assert.True(false, "Did not get a SPARQL Result Set as expected");
}
}
finally
{
PropertyFunctionFactory.RemoveFactory(factory);
provider.Dispose();
LuceneTestHarness.Index.Dispose();
}
}
/// <summary>
/// Creates a new Full Text Indexed Dataset
/// </summary>
/// <param name="dataset">Dataset to wrap</param>
/// <param name="indexer">Indexer to use</param>
/// <param name="indexNow">Whether the dataset provided should be indexed now, set to false if indexer is linked to an existing index for this data</param>
/// <remarks>
/// If <paramref name="indexNow"/> is true then the provided dataset will be fully indexed when this constructor is called
/// </remarks>
public FullTextIndexedDataset(ISparqlDataset dataset, IFullTextIndexer indexer, bool indexNow)
: base(dataset)
{
this._indexer = indexer;
//Index Now if requested
this._indexNow = indexNow;
if (indexNow)
{
foreach (IGraph g in this.Graphs)
{
this._indexer.Index(g);
}
}
}
public void FullTextIndexCreationLucenePredicates()
{
IFullTextIndexer indexer = null;
try
{
indexer = new LucenePredicatesIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema);
indexer.Index(this.GetTestData());
}
finally
{
if (indexer != null)
{
indexer.Dispose();
}
}
}
public void FullTextIndexSearchLucenePredicatesWithLimitAndThreshold()
{
IFullTextIndexer indexer = null;
IFullTextSearchProvider provider = null;
try
{
indexer = new LucenePredicatesIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema);
indexer.Index(this.GetTestData());
}
finally
{
if (indexer != null)
{
indexer.Dispose();
}
}
try
{
provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index);
NTriplesFormatter formatter = new NTriplesFormatter();
int i = 0;
foreach (IFullTextSearchResult result in provider.Match("http", 1.0d, 5))
{
i++;
Console.WriteLine(result.Node.ToString(formatter) + " - Scores " + result.Score);
Assert.IsTrue(result.Score >= 1.0d, "Score should be higher than desired threshold");
}
Assert.IsTrue(i <= 5, "Should be a max of 5 results");
}
finally
{
if (provider != null)
{
provider.Dispose();
}
}
}
private void EnableFullTextIndex()
{
if (this._dataset is FullTextIndexedDataset)
{
//Nothing to do
}
else if (this._dataset is WebDemandDataset)
{
WebDemandDataset ds = (WebDemandDataset)this._dataset;
this._dataset = ds.UnderlyingDataset;
this.EnableFullTextIndex();
this._dataset = new WebDemandDataset(this._dataset);
}
else
{
//Create and ensure index ready for use
this._ftIndex = new RAMDirectory();
var writer = new IndexWriter(this._ftIndex, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), IndexWriter.MaxFieldLength.UNLIMITED);
writer.Dispose();
//Create Indexer and wrap dataset
this._ftIndexer = new LuceneObjectsIndexer(this._ftIndex, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), new DefaultIndexSchema());
if (this._dataset is WebDemandDataset)
{
//Web Demand needs to go around Full Text as we want to index on demand loaded content
this._dataset = new WebDemandDataset(new FullTextIndexedDataset(((WebDemandDataset)this._dataset).UnderlyingDataset, this._ftIndexer, true));
}
else
{
this._dataset = new FullTextIndexedDataset(this._dataset, this._ftIndexer, true);
}
//Create and Register Optimizer
this._ftSearcher = new LuceneSearchProvider(Lucene.Net.Util.Version.LUCENE_29, this._ftIndex);
this._ftOptimiser = new FullTextOptimiser(this._ftSearcher);
SparqlOptimiser.AddOptimiser(this._ftOptimiser);
}
this._processor = new LeviathanQueryProcessor(this._dataset);
}
public void FullTextIndexSearchLuceneSubjectsWithThreshold()
{
IFullTextIndexer indexer = null;
IFullTextSearchProvider provider = null;
try
{
indexer = new LuceneSubjectsIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema);
indexer.Index(this.GetTestData());
}
finally
{
if (indexer != null)
{
indexer.Dispose();
}
}
try
{
provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index);
NTriplesFormatter formatter = new NTriplesFormatter();
foreach (IFullTextSearchResult result in provider.Match("http", 0.75d))
{
Console.WriteLine(result.Node.ToString(formatter) + " - Scores " + result.Score);
Assert.IsTrue(result.Score >= 0.75d, "Score should be higher than desired threshold");
}
}
finally
{
if (provider != null)
{
provider.Dispose();
}
}
}
public void FullTextIndexSearchLuceneObjects()
{
IFullTextIndexer indexer = null;
IFullTextSearchProvider provider = null;
try
{
indexer = new LuceneObjectsIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema);
indexer.Index(this.GetTestData());
}
finally
{
if (indexer != null)
{
indexer.Dispose();
}
}
try
{
provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index);
NTriplesFormatter formatter = new NTriplesFormatter();
foreach (IFullTextSearchResult result in provider.Match("http"))
{
Console.WriteLine(result.Node.ToString(formatter) + " - Scores " + result.Score);
}
}
finally
{
if (provider != null)
{
provider.Dispose();
}
}
}
/// <summary>
/// Tries to load an object based on information from the Configuration Graph
/// </summary>
/// <param name="g">Configuration Graph</param>
/// <param name="objNode">Object Node</param>
/// <param name="targetType">Target Type</param>
/// <param name="obj">Output Object</param>
/// <returns></returns>
public bool TryLoadObject(IGraph g, INode objNode, Type targetType, out object obj)
{
obj = null;
INode index = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "index"));
//INode indexer = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "indexer"));
INode searcher = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "searcher"));
INode analyzer = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "analyzer"));
INode schema = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "schema"));
INode version = g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "version"));
Object tempIndex, tempAnalyzer, tempSchema;
int ver = 2900;
//Always check for the version
ver = ConfigurationLoader.GetConfigurationInt32(g, objNode, version, 2900);
switch (targetType.FullName)
{
case DefaultIndexSchema:
obj = new DefaultIndexSchema();
break;
case FullTextOptimiser:
//Need to get the Search Provider
INode providerNode = ConfigurationLoader.GetConfigurationNode(g, objNode, searcher);
if (providerNode == null)
{
throw new DotNetRdfConfigurationException("Unable to load the Full Text Optimiser specified by the Node '" + objNode.ToString() + "' as there was no value specified for the required dnr-ft:searcher property");
}
Object tempSearcher = ConfigurationLoader.LoadObject(g, providerNode);
if (tempSearcher is IFullTextSearchProvider)
{
obj = new FullTextOptimiser((IFullTextSearchProvider)tempSearcher);
}
else
{
throw new DotNetRdfConfigurationException("Unable to load the Full Text Optimiser specified by the Node '" + objNode.ToString() + "' as the value specified for the dnr-ft:searcher property pointed to an object which could not be loaded as a type that implements the required IFullTextSearchProvider interface");
}
break;
case LuceneObjectsIndexer:
case LucenePredicatesIndexer:
case LuceneSubjectsIndexer:
case LuceneSearchProvider:
//For any Lucene Indexer/Search Provider need to know the Index, Analyzer and Schema to be used
//Then get the Index
tempIndex = ConfigurationLoader.GetConfigurationNode(g, objNode, index);
if (tempIndex == null)
{
throw new DotNetRdfConfigurationException("Unable to load the Lucene Indexer specified by the Node '" + objNode.ToString() + "' as there was no value specified for the required dnr-ft:index property");
}
tempIndex = ConfigurationLoader.LoadObject(g, (INode)tempIndex);
if (tempIndex is Directory)
{
//Next get the Analyzer (assume Standard if none specified)
tempAnalyzer = ConfigurationLoader.GetConfigurationNode(g, objNode, analyzer);
if (tempAnalyzer == null)
{
tempAnalyzer = new StandardAnalyzer(this.GetLuceneVersion(ver));
}
else
{
tempAnalyzer = ConfigurationLoader.LoadObject(g, (INode)tempAnalyzer);
}
if (tempAnalyzer is Analyzer)
{
//Finally get the Schema (assume Default if none specified)
tempSchema = ConfigurationLoader.GetConfigurationNode(g, objNode, schema);
if (tempSchema == null)
{
tempSchema = new DefaultIndexSchema();
}
else
{
tempSchema = ConfigurationLoader.LoadObject(g, (INode)tempSchema);
}
if (tempSchema is IFullTextIndexSchema)
{
//Now we can create the Object
switch (targetType.FullName)
{
case LuceneObjectsIndexer:
obj = new LuceneObjectsIndexer((Directory)tempIndex, (Analyzer)tempAnalyzer, (IFullTextIndexSchema)tempSchema);
break;
case LucenePredicatesIndexer:
obj = new LucenePredicatesIndexer((Directory)tempIndex, (Analyzer)tempAnalyzer, (IFullTextIndexSchema)tempSchema);
break;
case LuceneSubjectsIndexer:
obj = new LuceneSubjectsIndexer((Directory)tempIndex, (Analyzer)tempAnalyzer, (IFullTextIndexSchema)tempSchema);
break;
case LuceneSearchProvider:
//Before the Search Provider has been loaded determine whether we need to carry out auto-indexing
List <INode> sources = ConfigurationLoader.GetConfigurationData(g, objNode, g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "buildIndexFor"))).ToList();
if (sources.Count > 0)
{
//If there are sources to index ensure we have an indexer to index with
INode indexerNode = ConfigurationLoader.GetConfigurationNode(g, objNode, g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "buildIndexWith")));
if (indexerNode == null)
{
throw new DotNetRdfConfigurationException("Unable to load the Lucene Search Provider specified by the Node '" + objNode.ToString() + "' as there were values specified for the dnr-ft:buildIndexFor property but no dnr-ft:buildIndexWith property was found");
}
IFullTextIndexer indexer = ConfigurationLoader.LoadObject(g, indexerNode) as IFullTextIndexer;
if (indexer == null)
{
throw new DotNetRdfConfigurationException("Unable to load the Lucene Search Provider specified by the Node '" + objNode.ToString() + "' as the value given for the dnr-ft:buildIndexWith property pointed to an Object which could not be loaded as a type that implements the required IFullTextIndexer interface");
}
try
{
//For Each Source load it and Index it
foreach (INode sourceNode in sources)
{
Object source = ConfigurationLoader.LoadObject(g, sourceNode);
if (source is ISparqlDataset)
{
indexer.Index((ISparqlDataset)source);
}
else if (source is ITripleStore)
{
foreach (IGraph graph in ((ITripleStore)source).Graphs)
{
indexer.Index(graph);
}
}
else if (source is IGraph)
{
indexer.Index((IGraph)source);
}
else
{
throw new DotNetRdfConfigurationException("Unable to load the Lucene Search Provider specified by the Node '" + objNode.ToString() + "' as a value given for the dnr-ft:buildIndexFor property ('" + sourceNode.ToString() + "') pointed to an Object which could not be loaded as a type that implements one of the required interfaces: IGraph, ITripleStore or ISparqlDataset");
}
}
}
finally
{
indexer.Dispose();
}
}
//Then we actually load the Search Provider
obj = new LuceneSearchProvider(this.GetLuceneVersion(ver), (Directory)tempIndex, (Analyzer)tempAnalyzer, (IFullTextIndexSchema)tempSchema);
break;
}
}
else
{
throw new DotNetRdfConfigurationException("Unable to load the Lucene Indexer specified by the Node '" + objNode.ToString() + "' as the value given for the dnr-ft:schema property pointed to an Object which could not be loaded as a type that implements the required IFullTextIndexSchema interface");
}
}
else
{
throw new DotNetRdfConfigurationException("Unable to load the Lucene Indexer specified by the Node '" + objNode.ToString() + "' as the value given for the dnr-ft:analyzer property pointed to an Object which could not be loaded as a type that derives from the required Lucene.Net.Analysis.Analyzer type");
}
}
else
{
throw new DotNetRdfConfigurationException("Unable to load the Lucene Indexer specified by the Node '" + objNode.ToString() + "' as the value given for the dnr-ft:index property pointed to an Object which could not be loaded as a type that derives from the required Lucene.Net.Store.Directory type");
}
break;
default:
try
{
if (this._luceneAnalyzerType.IsAssignableFrom(targetType))
{
if (targetType.GetConstructor(new Type[] { typeof(LucVersion) }) != null)
{
obj = Activator.CreateInstance(targetType, new Object[] { this.GetLuceneVersion(ver) });
}
else
{
obj = Activator.CreateInstance(targetType);
}
}
else if (this._luceneDirectoryType.IsAssignableFrom(targetType))
{
String dir = ConfigurationLoader.GetConfigurationString(g, objNode, ConfigurationLoader.CreateConfigurationNode(g, ConfigurationLoader.PropertyFromFile));
if (dir != null)
{
try
{
obj = Activator.CreateInstance(targetType, new Object[] { dir });
}
catch
{
MethodInfo method = targetType.GetMethod("Open", new Type[] { typeof(DirInfo) });
if (method != null)
{
obj = method.Invoke(null, new Object[] { new DirInfo(ConfigurationLoader.ResolvePath(dir)) });
}
}
}
else
{
obj = Activator.CreateInstance(targetType);
}
//Ensure the Index if necessary
if (obj != null)
{
if (ConfigurationLoader.GetConfigurationBoolean(g, objNode, g.CreateUriNode(new Uri(FullTextHelper.FullTextConfigurationNamespace + "ensureIndex")), false))
{
IndexWriter writer = new IndexWriter((Directory)obj, new StandardAnalyzer(this.GetLuceneVersion(ver)));
writer.Close();
}
}
}
}
catch
{
//Since we know we don't allow loading of all analyzers and directories we allow for users to inject other object factories
//which may know how to load those specific instances
obj = null;
}
break;
}
return(obj != null);
}
private void ProcessResourceStream(IResource resource, IFullTextIndexer indexer)
{
indexer.AddDocumentFragment(resource.ID, GetResourceText(resource));
}
public void ProcessResourceText(int resID, IFullTextIndexer indexer)
{
IResource resource = _pluginEnvironment.ResourceStore.LoadResource(resID);
ProcessResourceStream(resource, indexer);
}
public void FullTextIndexDestructionLucenePredicates()
{
IFullTextIndexer indexer = null;
try
{
LuceneSearchProvider provider = null;
int origCount;
try
{
provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index);
origCount = provider.Match("http").Count();
provider.Dispose();
}
catch
{
origCount = 0;
}
finally
{
if (provider != null)
{
provider.Dispose();
}
}
Console.WriteLine("Prior to indexing search returns " + origCount + " result(s)");
indexer = new LucenePredicatesIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema);
IGraph g = this.GetTestData();
indexer.Index(g);
indexer.Dispose();
indexer = null;
int currCount;
try
{
provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index);
currCount = provider.Match("http").Count();
provider.Dispose();
}
catch
{
currCount = 0;
}
finally
{
if (provider != null)
{
provider.Dispose();
}
}
Console.WriteLine("After indexing search returns " + currCount + " result(s)");
indexer = new LucenePredicatesIndexer(LuceneTestHarness.Index, LuceneTestHarness.Analyzer, LuceneTestHarness.Schema);
indexer.Unindex(g);
indexer.Dispose();
indexer = null;
try
{
provider = new LuceneSearchProvider(LuceneTestHarness.LuceneVersion, LuceneTestHarness.Index);
currCount = provider.Match("http").Count();
Console.WriteLine("After unindexing search returns " + currCount + " result(s)");
}
finally
{
if (provider != null)
{
provider.Dispose();
}
}
Assert.Equal(origCount, currCount);
}
finally
{
if (indexer != null)
{
indexer.Dispose();
}
LuceneTestHarness.Index.Dispose();
}
}
/// <summary>
/// Creates a new Full Text Indexed Dataset
/// </summary>
/// <param name="dataset">Dataset to wrap</param>
/// <param name="indexer">Indexer to use</param>
/// <remarks>
/// Does not do any indexing, assumes the provided dataset is already indexed. When using this constructor only changes to the dataset affect the index
/// </remarks>
public FullTextIndexedDataset(ISparqlDataset dataset, IFullTextIndexer indexer)
: this(dataset, indexer, false)
{
}
private void RunTest(IFullTextIndexer indexer, String query, int expectedResults)
{
this.RunTest(indexer, query, expectedResults, true);
}
