private IInMemoryQueryableStore LoadInternal(Uri u, ExpansionProfile profile)
{
ExpansionContext context = new ExpansionContext(profile);
context.Uris.Enqueue(new UriToExpand(u, 0));
Expand(context);
return(context.Store);
}
private void ExpandByDatasets(UriToExpand u, ExpansionContext context)
{
if (u.Depth == context.Profile.MaxExpansionDepth)
{
return;
}
foreach (ExpansionDataset dataset in context.Profile.ExpansionDatasets)
{
this.ExpandByDataset(u, context, dataset);
}
}
private void ExpandByDataset(UriToExpand u, ExpansionContext context, ExpansionDataset dataset)
{
if (u.Depth == context.Profile.MaxExpansionDepth)
{
return;
}
if (dataset.Ignore)
{
return;
}
foreach (Uri endpoint in dataset.SparqlEndpoints)
{
Thread.Sleep(HttpRequestInterval);
SparqlRemoteEndpoint sparqlEndpoint = new SparqlRemoteEndpoint(endpoint);
try
{
SparqlParameterizedString queryString = new SparqlParameterizedString("DESCRIBE @uri");
queryString.SetUri("uri", u.Uri);
Object temp = sparqlEndpoint.QueryWithResultGraph(queryString.ToString());
if (temp is Graph)
{
Graph g = (Graph)temp;
this.ExpandGraph(u, g, context);
}
}
catch (RdfException rdfEx)
{
this.DebugErrors("Error: Tried to DESCRIBE <" + u.Uri.ToString() + "> against the SPARQL Endpoint <" + endpoint.ToString() + "> but an error occurred:", rdfEx);
}
catch (WebException webEx)
{
this.DebugErrors("Error: Tried to DESCRIBE <" + u.Uri.ToString() + "> against the SPARQL Endpoint <" + endpoint.ToString() + "> but an error occurred:", webEx);
}
}
foreach (Uri endpoint in dataset.UriLookupEndpoints)
{
this.ExpandByUriLookup(u, context, endpoint);
}
foreach (Uri endpoint in dataset.UriDiscoveryEndpoints)
{
this.ExpandByUriDiscovery(u, context, endpoint);
}
}
private void ExpandGraph(UriToExpand u, IGraph g, ExpansionContext context)
{
//Can ignore empty Graphs
//if (g.IsEmpty) return;
//If the Graph with the Base URI doesn't already exist we add it to the store
if (context.Store.HasGraph(g.BaseUri))
{
return;
}
context.Store.Add(g, true);
//If it didn't already exist we find URIs to expand upon from this Graph
this.FindExpandableUris(u, g, context);
//Then expand by Datasets
this.ExpandByDatasets(u, context);
}
private void ExpandByUriLookup(UriToExpand u, ExpansionContext context, Uri lookupEndpoint)
{
if (u.Depth == context.Profile.MaxExpansionDepth)
{
return;
}
StringBuilder requestUri = new StringBuilder();
requestUri.Append(lookupEndpoint.ToString());
requestUri.Append(Uri.EscapeDataString(u.Uri.ToString()));
Graph g = new Graph();
Thread.Sleep(HttpRequestInterval);
UriLoader.Load(g, new Uri(requestUri.ToString()));
this.ExpandGraph(u, g, context);
}
private void FindExpandableUris(UriToExpand u, IGraph g, ExpansionContext context)
{
if (u.Depth == context.Profile.MaxExpansionDepth)
{
return;
}
try
{
IUriNode current = g.CreateUriNode(u.Uri);
//Find owl:sameAs and rdfs:seeAlso links
foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, context.SameAs))
{
if (t.Object.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
}
foreach (Triple t in g.GetTriplesWithPredicateObject(context.SameAs, current))
{
if (t.Subject.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
}
foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, context.SeeAlso))
{
if (t.Object.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
}
//Then apply Linksets
foreach (ExpansionLinkset linkset in context.Profile.ExpansionLinksets)
{
//Check if Linkset is ignored
if (linkset.Ignore)
{
continue;
}
//Check if either Dataset is ignored
if (context.Profile.GetExpansionDataset(linkset.SubjectsTarget).Ignore)
{
return;
}
if (context.Profile.GetExpansionDataset(linkset.ObjectsTarget).Ignore)
{
return;
}
foreach (INode pred in linkset.Predicates)
{
//Find links from the current Graph Base URI
foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, pred))
{
if (t.Object.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
if (!linkset.IsDirected)
{
//If linkset is not directed then follow the link in the reverse direction
if (t.Subject.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
}
}
//Look for any links from other Graph URIs
foreach (IGraph graph in context.Store.Graphs)
{
INode curr = graph.CreateUriNode();
foreach (Triple t in g.GetTriplesWithSubjectPredicate(curr, pred))
{
if (t.Object.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
if (!linkset.IsDirected)
{
//If linkset is not directed then follow the link in the reverse direction
if (t.Subject.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
}
}
}
}
}
}
catch (Exception ex)
{
this.DebugErrors("Error: Trying to find links to follow from Graph '" + g.BaseUri.ToString() + "' when an error occurred", ex);
}
}
private void Expand(ExpansionContext context)
{
while (context.Uris.Count > 0)
{
//Get the next URI to expand upon and check it's not above the max expansion depth
//Or that it's already a Graph in the Store
UriToExpand u = context.GetNextUri();//context.Uris.Dequeue();
if (u == null)
{
return;
}
Debug.WriteLine("Expanding URI <" + u.Uri.ToString() + "> at Depth " + u.Depth);
Debug.WriteLine(context.Uris.Count + " remaining to expand");
Debug.WriteLine("Got " + context.Store.Graphs.Count + " Graphs so far");
if (u.Depth > context.Profile.MaxExpansionDepth)
{
continue;
}
if (context.Store.HasGraph(u.Uri))
{
continue;
}
//Try and retrieve RDF from the next URI
Graph g = new Graph();
try
{
UriLoader.Load(g, u.Uri);
}
catch (RdfException rdfEx)
{
//Ignore
this.DebugErrors("Error: Tried to expand URI <" + u.Uri.ToString() + "> but an RDF Error occurred", rdfEx);
}
catch (WebException webEx)
{
//Ignore
this.DebugErrors("Error: Tried to expand URI <" + u.Uri.ToString() + "> but a HTTP Error occurred", webEx);
}
ExpandGraph(u, g, context);
//If we've got any URIs to expand and we're not already multi-threading then spawn some
//threads to share out the work
if (!context.MultiThreading && context.Uris.Count > 0)
{
//REQ: Convert to an IAsyncResult pattern
context.MultiThreading = true;
List <Thread> threads = new List <Thread>();
for (int i = 0; i < Math.Min(context.Uris.Count, this._threadsToUse); i++)
{
threads.Add(new Thread(new ThreadStart(delegate { this.Expand(context); })));
}
threads.ForEach(t => t.Start());
while (threads.Any(t => t.ThreadState == System.Threading.ThreadState.Running))
{
Thread.Sleep(ThreadPollingInterval);
}
context.MultiThreading = false;
}
}
}
private void ExpandByUriLookup(UriToExpand u, ExpansionContext context, Uri lookupEndpoint)
{
if (u.Depth == context.Profile.MaxExpansionDepth) return;
StringBuilder requestUri = new StringBuilder();
requestUri.Append(lookupEndpoint.ToString());
requestUri.Append(Uri.EscapeDataString(u.Uri.ToString()));
Graph g = new Graph();
Thread.Sleep(HttpRequestInterval);
UriLoader.Load(g, new Uri(requestUri.ToString()));
this.ExpandGraph(u, g, context);
}
private void ExpandByDatasets(UriToExpand u, ExpansionContext context)
{
if (u.Depth == context.Profile.MaxExpansionDepth) return;
foreach (ExpansionDataset dataset in context.Profile.ExpansionDatasets)
{
this.ExpandByDataset(u, context, dataset);
}
}
private void ExpandByDataset(UriToExpand u, ExpansionContext context, ExpansionDataset dataset)
{
if (u.Depth == context.Profile.MaxExpansionDepth) return;
if (dataset.Ignore) return;
foreach (Uri endpoint in dataset.SparqlEndpoints)
{
Thread.Sleep(HttpRequestInterval);
SparqlRemoteEndpoint sparqlEndpoint = new SparqlRemoteEndpoint(endpoint);
try
{
SparqlParameterizedString queryString = new SparqlParameterizedString("DESCRIBE @uri");
queryString.SetUri("uri", u.Uri);
Object temp = sparqlEndpoint.QueryWithResultGraph(queryString.ToString());
if (temp is Graph)
{
Graph g = (Graph)temp;
this.ExpandGraph(u, g, context);
}
}
catch (RdfException rdfEx)
{
this.DebugErrors("Error: Tried to DESCRIBE <" + u.Uri.ToString() + "> against the SPARQL Endpoint <" + endpoint.ToString() + "> but an error occurred:", rdfEx);
}
catch (WebException webEx)
{
this.DebugErrors("Error: Tried to DESCRIBE <" + u.Uri.ToString() + "> against the SPARQL Endpoint <" + endpoint.ToString() + "> but an error occurred:", webEx);
}
}
foreach (Uri endpoint in dataset.UriLookupEndpoints)
{
this.ExpandByUriLookup(u, context, endpoint);
}
foreach (Uri endpoint in dataset.UriDiscoveryEndpoints)
{
this.ExpandByUriDiscovery(u, context, endpoint);
}
}
private void FindExpandableUris(UriToExpand u, IGraph g, ExpansionContext context)
{
if (u.Depth == context.Profile.MaxExpansionDepth) return;
try
{
IUriNode current = g.CreateUriNode(u.Uri);
//Find owl:sameAs and rdfs:seeAlso links
foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, context.SameAs))
{
if (t.Object.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
}
foreach (Triple t in g.GetTriplesWithPredicateObject(context.SameAs, current))
{
if (t.Subject.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
}
foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, context.SeeAlso))
{
if (t.Object.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
}
//Then apply Linksets
foreach (ExpansionLinkset linkset in context.Profile.ExpansionLinksets)
{
//Check if Linkset is ignored
if (linkset.Ignore) continue;
//Check if either Dataset is ignored
if (context.Profile.GetExpansionDataset(linkset.SubjectsTarget).Ignore) return;
if (context.Profile.GetExpansionDataset(linkset.ObjectsTarget).Ignore) return;
foreach (INode pred in linkset.Predicates)
{
//Find links from the current Graph Base URI
foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, pred))
{
if (t.Object.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
if (!linkset.IsDirected)
{
//If linkset is not directed then follow the link in the reverse direction
if (t.Subject.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
}
}
//Look for any links from other Graph URIs
foreach (IGraph graph in context.Store.Graphs)
{
INode curr = graph.CreateUriNode();
foreach (Triple t in g.GetTriplesWithSubjectPredicate(curr, pred))
{
if (t.Object.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
if (!linkset.IsDirected)
{
//If linkset is not directed then follow the link in the reverse direction
if (t.Subject.NodeType == NodeType.Uri)
{
context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
}
}
}
}
}
}
}
catch (Exception ex)
{
this.DebugErrors("Error: Trying to find links to follow from Graph '" + g.BaseUri.ToString() + "' when an error occurred", ex);
}
}
private void ExpandGraph(UriToExpand u, IGraph g, ExpansionContext context)
{
//Can ignore empty Graphs
//if (g.IsEmpty) return;
//If the Graph with the Base URI doesn't already exist we add it to the store
if (context.Store.HasGraph(g.BaseUri)) return;
context.Store.Add(g, true);
//If it didn't already exist we find URIs to expand upon from this Graph
this.FindExpandableUris(u, g, context);
//Then expand by Datasets
this.ExpandByDatasets(u, context);
}
private void Expand(ExpansionContext context)
{
while (context.Uris.Count > 0)
{
//Get the next URI to expand upon and check it's not above the max expansion depth
//Or that it's already a Graph in the Store
UriToExpand u = context.GetNextUri();//context.Uris.Dequeue();
if (u == null) return;
Debug.WriteLine("Expanding URI <" + u.Uri.ToString() + "> at Depth " + u.Depth);
Debug.WriteLine(context.Uris.Count + " remaining to expand");
Debug.WriteLine("Got " + context.Store.Graphs.Count + " Graphs so far");
if (u.Depth > context.Profile.MaxExpansionDepth) continue;
if (context.Store.HasGraph(u.Uri)) continue;
//Try and retrieve RDF from the next URI
Graph g = new Graph();
try
{
UriLoader.Load(g, u.Uri);
}
catch (RdfException rdfEx)
{
//Ignore
this.DebugErrors("Error: Tried to expand URI <" + u.Uri.ToString() + "> but an RDF Error occurred", rdfEx);
}
catch (WebException webEx)
{
//Ignore
this.DebugErrors("Error: Tried to expand URI <" + u.Uri.ToString() + "> but a HTTP Error occurred", webEx);
}
ExpandGraph(u, g, context);
//If we've got any URIs to expand and we're not already multi-threading then spawn some
//threads to share out the work
if (!context.MultiThreading && context.Uris.Count > 0)
{
//REQ: Convert to an IAsyncResult pattern
context.MultiThreading = true;
List<Thread> threads = new List<Thread>();
for (int i = 0; i < Math.Min(context.Uris.Count, this._threadsToUse); i++)
{
threads.Add(new Thread(new ThreadStart(delegate { this.Expand(context); })));
}
threads.ForEach(t => t.Start());
while (threads.Any(t => t.ThreadState == System.Threading.ThreadState.Running))
{
Thread.Sleep(ThreadPollingInterval);
}
context.MultiThreading = false;
}
}
}
private IInMemoryQueryableStore LoadInternal(Uri u, ExpansionProfile profile)
{
ExpansionContext context = new ExpansionContext(profile);
context.Uris.Enqueue(new UriToExpand(u, 0));
Expand(context);
return context.Store;
}
private void ExpandByUriDiscovery(UriToExpand u, ExpansionContext context, Uri discoveryEndpoint)
{
if (u.Depth == context.Profile.MaxExpansionDepth) return;
StringBuilder requestUri = new StringBuilder();
requestUri.Append(discoveryEndpoint.ToString());
requestUri.Append(Uri.EscapeDataString(u.Uri.ToString()));
Graph g = new Graph();
UriLoader.Load(g, new Uri(requestUri.ToString()));
this.FindExpandableUris(u, g, context);
}