Example #1
0
        private IInMemoryQueryableStore LoadInternal(Uri u, ExpansionProfile profile)
        {
            ExpansionContext context = new ExpansionContext(profile);

            context.Uris.Enqueue(new UriToExpand(u, 0));

            Expand(context);

            return(context.Store);
        }
Example #2
0
        private void ExpandByDatasets(UriToExpand u, ExpansionContext context)
        {
            if (u.Depth == context.Profile.MaxExpansionDepth)
            {
                return;
            }

            foreach (ExpansionDataset dataset in context.Profile.ExpansionDatasets)
            {
                this.ExpandByDataset(u, context, dataset);
            }
        }
Example #3
0
        private void ExpandByDataset(UriToExpand u, ExpansionContext context, ExpansionDataset dataset)
        {
            if (u.Depth == context.Profile.MaxExpansionDepth)
            {
                return;
            }
            if (dataset.Ignore)
            {
                return;
            }

            foreach (Uri endpoint in dataset.SparqlEndpoints)
            {
                Thread.Sleep(HttpRequestInterval);
                SparqlRemoteEndpoint sparqlEndpoint = new SparqlRemoteEndpoint(endpoint);
                try
                {
                    SparqlParameterizedString queryString = new SparqlParameterizedString("DESCRIBE @uri");
                    queryString.SetUri("uri", u.Uri);
                    Object temp = sparqlEndpoint.QueryWithResultGraph(queryString.ToString());
                    if (temp is Graph)
                    {
                        Graph g = (Graph)temp;
                        this.ExpandGraph(u, g, context);
                    }
                }
                catch (RdfException rdfEx)
                {
                    this.DebugErrors("Error: Tried to DESCRIBE <" + u.Uri.ToString() + "> against the SPARQL Endpoint <" + endpoint.ToString() + "> but an error occurred:", rdfEx);
                }
                catch (WebException webEx)
                {
                    this.DebugErrors("Error: Tried to DESCRIBE <" + u.Uri.ToString() + "> against the SPARQL Endpoint <" + endpoint.ToString() + "> but an error occurred:", webEx);
                }
            }

            foreach (Uri endpoint in dataset.UriLookupEndpoints)
            {
                this.ExpandByUriLookup(u, context, endpoint);
            }

            foreach (Uri endpoint in dataset.UriDiscoveryEndpoints)
            {
                this.ExpandByUriDiscovery(u, context, endpoint);
            }
        }
Example #4
0
        private void ExpandGraph(UriToExpand u, IGraph g, ExpansionContext context)
        {
            //Can ignore empty Graphs
            //if (g.IsEmpty) return;

            //If the Graph with the Base URI doesn't already exist we add it to the store
            if (context.Store.HasGraph(g.BaseUri))
            {
                return;
            }
            context.Store.Add(g, true);

            //If it didn't already exist we find URIs to expand upon from this Graph
            this.FindExpandableUris(u, g, context);

            //Then expand by Datasets
            this.ExpandByDatasets(u, context);
        }
Example #5
0
        private void ExpandByUriLookup(UriToExpand u, ExpansionContext context, Uri lookupEndpoint)
        {
            if (u.Depth == context.Profile.MaxExpansionDepth)
            {
                return;
            }

            StringBuilder requestUri = new StringBuilder();

            requestUri.Append(lookupEndpoint.ToString());
            requestUri.Append(Uri.EscapeDataString(u.Uri.ToString()));

            Graph g = new Graph();

            Thread.Sleep(HttpRequestInterval);
            UriLoader.Load(g, new Uri(requestUri.ToString()));

            this.ExpandGraph(u, g, context);
        }
Example #6
0
        private void FindExpandableUris(UriToExpand u, IGraph g, ExpansionContext context)
        {
            if (u.Depth == context.Profile.MaxExpansionDepth)
            {
                return;
            }

            try
            {
                IUriNode current = g.CreateUriNode(u.Uri);

                //Find owl:sameAs and rdfs:seeAlso links
                foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, context.SameAs))
                {
                    if (t.Object.NodeType == NodeType.Uri)
                    {
                        context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
                        context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                    }
                }
                foreach (Triple t in g.GetTriplesWithPredicateObject(context.SameAs, current))
                {
                    if (t.Subject.NodeType == NodeType.Uri)
                    {
                        context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
                        context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                    }
                }
                foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, context.SeeAlso))
                {
                    if (t.Object.NodeType == NodeType.Uri)
                    {
                        context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
                        context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                    }
                }

                //Then apply Linksets
                foreach (ExpansionLinkset linkset in context.Profile.ExpansionLinksets)
                {
                    //Check if Linkset is ignored
                    if (linkset.Ignore)
                    {
                        continue;
                    }

                    //Check if either Dataset is ignored
                    if (context.Profile.GetExpansionDataset(linkset.SubjectsTarget).Ignore)
                    {
                        return;
                    }
                    if (context.Profile.GetExpansionDataset(linkset.ObjectsTarget).Ignore)
                    {
                        return;
                    }

                    foreach (INode pred in linkset.Predicates)
                    {
                        //Find links from the current Graph Base URI
                        foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, pred))
                        {
                            if (t.Object.NodeType == NodeType.Uri)
                            {
                                context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
                                context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                            }
                            if (!linkset.IsDirected)
                            {
                                //If linkset is not directed then follow the link in the reverse direction
                                if (t.Subject.NodeType == NodeType.Uri)
                                {
                                    context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
                                    context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                                }
                            }
                        }

                        //Look for any links from other Graph URIs
                        foreach (IGraph graph in context.Store.Graphs)
                        {
                            INode curr = graph.CreateUriNode();
                            foreach (Triple t in g.GetTriplesWithSubjectPredicate(curr, pred))
                            {
                                if (t.Object.NodeType == NodeType.Uri)
                                {
                                    context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
                                    context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                                }
                                if (!linkset.IsDirected)
                                {
                                    //If linkset is not directed then follow the link in the reverse direction
                                    if (t.Subject.NodeType == NodeType.Uri)
                                    {
                                        context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
                                        context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                                    }
                                }
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                this.DebugErrors("Error: Trying to find links to follow from Graph '" + g.BaseUri.ToString() + "' when an error occurred", ex);
            }
        }
Example #7
0
        private void Expand(ExpansionContext context)
        {
            while (context.Uris.Count > 0)
            {
                //Get the next URI to expand upon and check it's not above the max expansion depth
                //Or that it's already a Graph in the Store
                UriToExpand u = context.GetNextUri();//context.Uris.Dequeue();
                if (u == null)
                {
                    return;
                }

                Debug.WriteLine("Expanding URI <" + u.Uri.ToString() + "> at Depth " + u.Depth);
                Debug.WriteLine(context.Uris.Count + " remaining to expand");
                Debug.WriteLine("Got " + context.Store.Graphs.Count + " Graphs so far");

                if (u.Depth > context.Profile.MaxExpansionDepth)
                {
                    continue;
                }
                if (context.Store.HasGraph(u.Uri))
                {
                    continue;
                }

                //Try and retrieve RDF from the next URI
                Graph g = new Graph();
                try
                {
                    UriLoader.Load(g, u.Uri);
                }
                catch (RdfException rdfEx)
                {
                    //Ignore
                    this.DebugErrors("Error: Tried to expand URI <" + u.Uri.ToString() + "> but an RDF Error occurred", rdfEx);
                }
                catch (WebException webEx)
                {
                    //Ignore
                    this.DebugErrors("Error: Tried to expand URI <" + u.Uri.ToString() + "> but a HTTP Error occurred", webEx);
                }

                ExpandGraph(u, g, context);

                //If we've got any URIs to expand and we're not already multi-threading then spawn some
                //threads to share out the work
                if (!context.MultiThreading && context.Uris.Count > 0)
                {
                    //REQ: Convert to an IAsyncResult pattern
                    context.MultiThreading = true;
                    List <Thread> threads = new List <Thread>();
                    for (int i = 0; i < Math.Min(context.Uris.Count, this._threadsToUse); i++)
                    {
                        threads.Add(new Thread(new ThreadStart(delegate { this.Expand(context); })));
                    }
                    threads.ForEach(t => t.Start());
                    while (threads.Any(t => t.ThreadState == System.Threading.ThreadState.Running))
                    {
                        Thread.Sleep(ThreadPollingInterval);
                    }
                    context.MultiThreading = false;
                }
            }
        }
Example #8
0
        private void ExpandByUriLookup(UriToExpand u, ExpansionContext context, Uri lookupEndpoint)
        {
            if (u.Depth == context.Profile.MaxExpansionDepth) return;

            StringBuilder requestUri = new StringBuilder();
            requestUri.Append(lookupEndpoint.ToString());
            requestUri.Append(Uri.EscapeDataString(u.Uri.ToString()));

            Graph g = new Graph();
            Thread.Sleep(HttpRequestInterval);
            UriLoader.Load(g, new Uri(requestUri.ToString()));

            this.ExpandGraph(u, g, context);
        }
Example #9
0
        private void ExpandByDatasets(UriToExpand u, ExpansionContext context)
        {
            if (u.Depth == context.Profile.MaxExpansionDepth) return;

            foreach (ExpansionDataset dataset in context.Profile.ExpansionDatasets)
            {
                this.ExpandByDataset(u, context, dataset);
            }
        }
Example #10
0
        private void ExpandByDataset(UriToExpand u, ExpansionContext context, ExpansionDataset dataset)
        {
            if (u.Depth == context.Profile.MaxExpansionDepth) return;
            if (dataset.Ignore) return;

            foreach (Uri endpoint in dataset.SparqlEndpoints)
            {
                Thread.Sleep(HttpRequestInterval);
                SparqlRemoteEndpoint sparqlEndpoint = new SparqlRemoteEndpoint(endpoint);
                try
                {
                    SparqlParameterizedString queryString = new SparqlParameterizedString("DESCRIBE @uri");
                    queryString.SetUri("uri", u.Uri);
                    Object temp = sparqlEndpoint.QueryWithResultGraph(queryString.ToString());
                    if (temp is Graph)
                    {
                        Graph g = (Graph)temp;
                        this.ExpandGraph(u, g, context);
                    }
                }
                catch (RdfException rdfEx)
                {
                    this.DebugErrors("Error: Tried to DESCRIBE <" + u.Uri.ToString() + "> against the SPARQL Endpoint <" + endpoint.ToString() + "> but an error occurred:", rdfEx);
                }
                catch (WebException webEx)
                {
                    this.DebugErrors("Error: Tried to DESCRIBE <" + u.Uri.ToString() + "> against the SPARQL Endpoint <" + endpoint.ToString() + "> but an error occurred:", webEx);
                }
            }

            foreach (Uri endpoint in dataset.UriLookupEndpoints)
            {
                this.ExpandByUriLookup(u, context, endpoint);
            }

            foreach (Uri endpoint in dataset.UriDiscoveryEndpoints)
            {
                this.ExpandByUriDiscovery(u, context, endpoint);
            }
        }
Example #11
0
        private void FindExpandableUris(UriToExpand u, IGraph g, ExpansionContext context)
        {
            if (u.Depth == context.Profile.MaxExpansionDepth) return;

            try
            {
                IUriNode current = g.CreateUriNode(u.Uri);

                //Find owl:sameAs and rdfs:seeAlso links
                foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, context.SameAs))
                {
                    if (t.Object.NodeType == NodeType.Uri)
                    {
                        context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
                        context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                    }
                }
                foreach (Triple t in g.GetTriplesWithPredicateObject(context.SameAs, current))
                {
                    if (t.Subject.NodeType == NodeType.Uri)
                    {
                        context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
                        context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                    }
                }
                foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, context.SeeAlso))
                {
                    if (t.Object.NodeType == NodeType.Uri)
                    {
                        context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
                        context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                    }
                }

                //Then apply Linksets
                foreach (ExpansionLinkset linkset in context.Profile.ExpansionLinksets)
                {
                    //Check if Linkset is ignored
                    if (linkset.Ignore) continue;

                    //Check if either Dataset is ignored
                    if (context.Profile.GetExpansionDataset(linkset.SubjectsTarget).Ignore) return;
                    if (context.Profile.GetExpansionDataset(linkset.ObjectsTarget).Ignore) return;

                    foreach (INode pred in linkset.Predicates)
                    {
                        //Find links from the current Graph Base URI
                        foreach (Triple t in g.GetTriplesWithSubjectPredicate(current, pred))
                        {
                            if (t.Object.NodeType == NodeType.Uri)
                            {
                                context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
                                context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                            }
                            if (!linkset.IsDirected)
                            {
                                //If linkset is not directed then follow the link in the reverse direction
                                if (t.Subject.NodeType == NodeType.Uri)
                                {
                                    context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
                                    context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                                }
                            }
                        }

                        //Look for any links from other Graph URIs
                        foreach (IGraph graph in context.Store.Graphs)
                        {
                            INode curr = graph.CreateUriNode();
                            foreach (Triple t in g.GetTriplesWithSubjectPredicate(curr, pred))
                            {
                                if (t.Object.NodeType == NodeType.Uri)
                                {
                                    context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Object).Uri, u.Depth + 1));
                                    context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                                }
                                if (!linkset.IsDirected)
                                {
                                    //If linkset is not directed then follow the link in the reverse direction
                                    if (t.Subject.NodeType == NodeType.Uri)
                                    {
                                        context.Uris.Enqueue(new UriToExpand(((IUriNode)t.Subject).Uri, u.Depth + 1));
                                        context.LinkGraph.Assert(t.CopyTriple(context.LinkGraph));
                                    }
                                }
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                this.DebugErrors("Error: Trying to find links to follow from Graph '" + g.BaseUri.ToString() + "' when an error occurred", ex);
            }
        }
Example #12
0
        private void ExpandGraph(UriToExpand u, IGraph g, ExpansionContext context)
        {
            //Can ignore empty Graphs
            //if (g.IsEmpty) return;

            //If the Graph with the Base URI doesn't already exist we add it to the store
            if (context.Store.HasGraph(g.BaseUri)) return;
            context.Store.Add(g, true);

            //If it didn't already exist we find URIs to expand upon from this Graph
            this.FindExpandableUris(u, g, context);

            //Then expand by Datasets
            this.ExpandByDatasets(u, context);
        }
Example #13
0
        private void Expand(ExpansionContext context)
        {
            while (context.Uris.Count > 0)
            {
                //Get the next URI to expand upon and check it's not above the max expansion depth
                //Or that it's already a Graph in the Store
                UriToExpand u = context.GetNextUri();//context.Uris.Dequeue();
                if (u == null) return;

                Debug.WriteLine("Expanding URI <" + u.Uri.ToString() + "> at Depth " + u.Depth);
                Debug.WriteLine(context.Uris.Count + " remaining to expand");
                Debug.WriteLine("Got " + context.Store.Graphs.Count + " Graphs so far");

                if (u.Depth > context.Profile.MaxExpansionDepth) continue;
                if (context.Store.HasGraph(u.Uri)) continue;

                //Try and retrieve RDF from the next URI
                Graph g = new Graph();
                try
                {
                    UriLoader.Load(g, u.Uri);
                }
                catch (RdfException rdfEx)
                {
                    //Ignore
                    this.DebugErrors("Error: Tried to expand URI <" + u.Uri.ToString() + "> but an RDF Error occurred", rdfEx);
                }
                catch (WebException webEx)
                {
                    //Ignore
                    this.DebugErrors("Error: Tried to expand URI <" + u.Uri.ToString() + "> but a HTTP Error occurred", webEx); 
                }

                ExpandGraph(u, g, context);

                //If we've got any URIs to expand and we're not already multi-threading then spawn some
                //threads to share out the work
                if (!context.MultiThreading && context.Uris.Count > 0)
                {
                    //REQ: Convert to an IAsyncResult pattern
                    context.MultiThreading = true;
                    List<Thread> threads = new List<Thread>();
                    for (int i = 0; i < Math.Min(context.Uris.Count, this._threadsToUse); i++)
                    {
                        threads.Add(new Thread(new ThreadStart(delegate { this.Expand(context); })));
                    }
                    threads.ForEach(t => t.Start());
                    while (threads.Any(t => t.ThreadState == System.Threading.ThreadState.Running)) 
                    {
                        Thread.Sleep(ThreadPollingInterval);
                    }
                    context.MultiThreading = false;
                }
            }
        }
Example #14
0
        private IInMemoryQueryableStore LoadInternal(Uri u, ExpansionProfile profile)
        {
            ExpansionContext context = new ExpansionContext(profile);
            context.Uris.Enqueue(new UriToExpand(u, 0));

            Expand(context);

            return context.Store;
        }
        private void ExpandByUriDiscovery(UriToExpand u, ExpansionContext context, Uri discoveryEndpoint)
        {
            if (u.Depth == context.Profile.MaxExpansionDepth) return;

            StringBuilder requestUri = new StringBuilder();
            requestUri.Append(discoveryEndpoint.ToString());
            requestUri.Append(Uri.EscapeDataString(u.Uri.ToString()));

            Graph g = new Graph();
            UriLoader.Load(g, new Uri(requestUri.ToString()));

            this.FindExpandableUris(u, g, context);
        }