public void ParsingMultiHandlerGraphWithFactory()
{
EnsureTestData();
Graph g = new Graph();
GraphHandler h = new GraphHandler(g);
MultiHandler handler = new MultiHandler(new IRdfHandler[] { h }, g);
TurtleParser parser = new TurtleParser();
parser.Load(handler, "multi_handler_tests_temp.ttl");
Assert.Same(g, g.Triples.First().Graph);
}
public void ParsingMultiHandlerGraphAndNull()
{
EnsureTestData();
Graph g = new Graph();
GraphHandler handler1 = new GraphHandler(g);
NullHandler handler2 = new NullHandler();
MultiHandler handler = new MultiHandler(new IRdfHandler[] { handler1, handler2 });
TurtleParser parser = new TurtleParser();
parser.Load(handler, "multi_handler_tests_temp.ttl");
}
public void ParsingMultiHandlerGraphAndCount()
{
EnsureTestData();
Graph g = new Graph();
GraphHandler handler1 = new GraphHandler(g);
CountHandler handler2 = new CountHandler();
MultiHandler handler = new MultiHandler(new IRdfHandler[] { handler1, handler2 });
TurtleParser parser = new TurtleParser();
parser.Load(handler, "multi_handler_tests_temp.ttl");
Assert.Equal(g.Triples.Count, handler2.Count);
}
public void ParsingMultiHandlerGraphAndCount()
{
EnsureTestData();
Graph g = new Graph();
GraphHandler handler1 = new GraphHandler(g);
CountHandler handler2 = new CountHandler();
MultiHandler handler = new MultiHandler(new IRdfHandler[] { handler1, handler2 });
TurtleParser parser = new TurtleParser();
parser.Load(handler, "temp.ttl");
Assert.AreEqual(g.Triples.Count, handler2.Count, "Expected Counts to be the same");
}
public void ParsingMultiHandlerTwoGraphs()
{
EnsureTestData();
Graph g = new Graph();
Graph h = new Graph();
GraphHandler handler1 = new GraphHandler(g);
GraphHandler handler2 = new GraphHandler(h);
MultiHandler handler = new MultiHandler(new IRdfHandler[] { handler1, handler2 });
TurtleParser parser = new TurtleParser();
parser.Load(handler, "temp.ttl");
Assert.AreEqual(g.Triples.Count, h.Triples.Count, "Expected same number of Triples");
Assert.AreEqual(g, h, "Expected Graphs to be equal");
}
public void ParsingMultiHandlerTwoGraphs()
{
EnsureTestData();
Graph g = new Graph();
Graph h = new Graph();
GraphHandler handler1 = new GraphHandler(g);
GraphHandler handler2 = new GraphHandler(h);
MultiHandler handler = new MultiHandler(new IRdfHandler[] { handler1, handler2 });
TurtleParser parser = new TurtleParser();
parser.Load(handler, "temp.ttl");
Assert.AreEqual(g.Triples.Count, h.Triples.Count, "Expected same number of Triples");
Assert.AreEqual(g, h, "Expected Graphs to be equal");
}
public void ParsingMultiHandlerGraphAndPaging()
{
EnsureTestData();
Graph g = new Graph();
Graph h = new Graph();
GraphHandler handler1 = new GraphHandler(g);
PagingHandler handler2 = new PagingHandler(new GraphHandler(h), 100);
MultiHandler handler = new MultiHandler(new IRdfHandler[] { handler1, handler2 });
TurtleParser parser = new TurtleParser();
parser.Load(handler, "temp.ttl");
Assert.AreEqual(101, g.Triples.Count, "Triples should have been limited to 101 (1st Graph)");
Assert.AreEqual(100, h.Triples.Count, "Triples should have been limited to 100 (2nd Graph)");
Assert.AreNotEqual(g.Triples.Count, h.Triples.Count, "Expected different number of Triples");
Assert.AreNotEqual(g, h, "Expected Graphs to not be equal");
}
public void ParsingMultiHandlerGraphAndPaging()
{
EnsureTestData();
Graph g = new Graph();
Graph h = new Graph();
GraphHandler handler1 = new GraphHandler(g);
PagingHandler handler2 = new PagingHandler(new GraphHandler(h), 100);
MultiHandler handler = new MultiHandler(new IRdfHandler[] { handler1, handler2 });
TurtleParser parser = new TurtleParser();
parser.Load(handler, "multi_handler_tests_temp.ttl");
Assert.Equal(101, g.Triples.Count);
Assert.Equal(100, h.Triples.Count);
Assert.NotEqual(g.Triples.Count, h.Triples.Count);
Assert.NotEqual(g, h);
}
public void ParsingMultiHandlerGraphAndPaging()
{
EnsureTestData();
Graph g = new Graph();
Graph h = new Graph();
GraphHandler handler1 = new GraphHandler(g);
PagingHandler handler2 = new PagingHandler(new GraphHandler(h), 100);
MultiHandler handler = new MultiHandler(new IRdfHandler[] { handler1, handler2 });
TurtleParser parser = new TurtleParser();
parser.Load(handler, "temp.ttl");
Assert.AreEqual(101, g.Triples.Count, "Triples should have been limited to 101 (1st Graph)");
Assert.AreEqual(100, h.Triples.Count, "Triples should have been limited to 100 (2nd Graph)");
Assert.AreNotEqual(g.Triples.Count, h.Triples.Count, "Expected different number of Triples");
Assert.AreNotEqual(g, h, "Expected Graphs to not be equal");
}
public void ParsingMultiHandlerBadInstantiation2()
{
MultiHandler handler = new MultiHandler(null);
}
public void ParsingMultiHandlerBadInstantiation()
{
MultiHandler handler = new MultiHandler(Enumerable.Empty<IRdfHandler>());
}
public IRdfHandler ToCache(Uri requestUri, Uri responseUri, String etag)
{
IRdfHandler handler = null;
try
{
bool cacheTwice = !requestUri.ToString().Equals(responseUri.ToString(), StringComparison.OrdinalIgnoreCase);
//Cache the ETag if present
if (this._canCacheETag && etag != null && !etag.Equals(String.Empty))
{
int id = requestUri.GetEnhancedHashCode();
bool requireAdd = false;
if (this._etags.ContainsKey(id))
{
if (!this._etags[id].Equals(etag))
{
//If the ETag has changed remove it and then re-add it
this.RemoveETag(requestUri);
requireAdd = true;
}
}
else
{
requireAdd = true;
}
if (requireAdd)
{
//Add a New ETag
this._etags.Add(id, etag);
using (StreamWriter writer = new StreamWriter(this._etagFile, true, Encoding.UTF8))
{
writer.WriteLine(id + "\t" + etag);
writer.Close();
}
}
//Cache under the Response URI as well if applicable
if (cacheTwice)
{
id = responseUri.GetEnhancedHashCode();
requireAdd = false;
if (this._etags.ContainsKey(id))
{
if (!this._etags[id].Equals(etag))
{
//If the ETag has changed remove it and then re-add it
this.RemoveETag(responseUri);
requireAdd = true;
}
}
else
{
requireAdd = true;
}
if (requireAdd)
{
using (StreamWriter writer = new StreamWriter(this._etagFile, true, Encoding.UTF8))
{
writer.WriteLine(id + "\t" + etag);
writer.Close();
}
}
}
}
//Then if we are caching Graphs return WriteThroughHandlers to do the caching for us
if (this._canCacheGraphs)
{
String graph = Path.Combine(this._graphDir, requestUri.GetSha256Hash());
handler = new WriteThroughHandler(this._formatterType, new StreamWriter(graph), true);
if (cacheTwice)
{
graph = Path.Combine(this._graphDir, responseUri.GetSha256Hash());
handler = new MultiHandler(new IRdfHandler[] { handler, new WriteThroughHandler(this._formatterType, new StreamWriter(graph), true) });
}
}
}
catch (IOException)
{
//Ignore - if we get an IO Exception we failed to cache somehow
}
catch (RdfOutputException)
{
//Ignore - if we get an RDF Output Exception then we failed to cache
}
return handler;
}
public void Run()
{
if (this._args.Length == 0)
{
this.ShowUsage();
}
else
{
if (!this.ParseOptions())
{
Console.Error.WriteLine("rdfOptStats: Error: One/More options were invalid");
return;
}
if (this._inputs.Count == 0)
{
Console.Error.WriteLine("rdfOptStats: Error: No Inputs Specified");
return;
}
List<BaseStatsHandler> handlers = new List<BaseStatsHandler>();
if (this._subjects && this._predicates && this._objects)
{
handlers.Add(new SPOStatsHandler(this._literals));
}
else if (this._subjects && this._predicates)
{
handlers.Add(new SPStatsHandler(this._literals));
}
else
{
if (this._subjects) handlers.Add(new SubjectStatsHandler(this._literals));
if (this._predicates) handlers.Add(new PredicateStatsHandler(this._literals));
if (this._objects) handlers.Add(new ObjectStatsHandler(this._literals));
}
if (this._nodes)
{
handlers.Add(new NodeStatsHandler());
}
bool ok = true;
IRdfHandler handler;
if (handlers.Count == 1)
{
handler = handlers[0];
}
else
{
handler = new MultiHandler(handlers.OfType<IRdfHandler>());
}
Stopwatch timer = new Stopwatch();
timer.Start();
for (int i = 0; i < this._inputs.Count; i++)
{
Console.WriteLine("rdfOptStats: Processing Input " + (i + 1) + " of " + this._inputs.Count + " - '" + this._inputs[i] + "'");
try
{
FileLoader.Load(handler, this._inputs[i]);
}
catch (RdfParserSelectionException selEx)
{
ok = false;
Console.Error.WriteLine("rdfOptStats: Error: Unable to select a Parser to read input");
break;
}
catch (RdfParseException parseEx)
{
ok = false;
Console.Error.WriteLine("rdfOptStats: Error: Parsing Error while reading input");
break;
}
catch (RdfException parseEx)
{
ok = false;
Console.Error.WriteLine("rdfOptStats: Error: RDF Error while reading input");
break;
}
catch (Exception ex)
{
ok = false;
Console.Error.WriteLine("rdfOptStats: Error: Unexpected Error while reading input");
break;
}
}
Console.WriteLine("rdfOptStats: Finished Processing Inputs");
timer.Stop();
Console.WriteLine("rdfOptStats: Took " + timer.Elapsed + " to process inputs");
timer.Reset();
if (ok)
{
//Output the Stats
timer.Start();
Graph g = new Graph();
g.NamespaceMap.Import(handlers.First().Namespaces);
try
{
foreach (BaseStatsHandler h in handlers)
{
h.GetStats(g);
}
IRdfWriter writer = MimeTypesHelper.GetWriter(MimeTypesHelper.GetMimeTypes(Path.GetExtension(this._file)));
if (writer is ICompressingWriter)
{
((ICompressingWriter)writer).CompressionLevel = WriterCompressionLevel.High;
}
if (writer is IHighSpeedWriter)
{
((IHighSpeedWriter)writer).HighSpeedModePermitted = false;
}
writer.Save(g, this._file);
Console.WriteLine("rdfOptStats: Statistics output to " + this._file);
timer.Stop();
Console.WriteLine("rdfOptStats: Took " + timer.Elapsed + " to output statistics");
}
catch (Exception ex)
{
Console.Error.WriteLine("rdfOptStats: Error: Unexpected error outputting statistics to " + this._file);
Console.Error.WriteLine(ex.Message);
Console.Error.WriteLine(ex.StackTrace);
}
}
else
{
Console.Error.WriteLine("rdfOptStats: Error: Unable to output statistics due to errors during input processing");
}
}
}
/// <summary>
/// Attempts to load a RDF Graph from the given URI using a RDF Handler
/// </summary>
/// <param name="handler">RDF Handler to use</param>
/// <param name="u">URI to attempt to get RDF from</param>
/// <param name="parser">Parser to use</param>
/// <remarks>
/// <para>
/// Uses the supplied parser to attempt parsing regardless of the actual Content Type returned
/// </para>
/// <para>
/// In the event that the URI is a File URI the <see cref="FileLoader">FileLoader</see> will be used instead
/// </para>
/// <para>
/// If the URI is a Data URI then the <see cref="DataUriLoader">DataUriLoader</see> will be used instead.
/// </para>
/// </remarks>
public static void Load(IRdfHandler handler, Uri u, IRdfReader parser)
{
if (handler == null) throw new RdfParseException("Cannot read RDF using a null RDF Handler");
if (u == null) throw new RdfParseException("Cannot load RDF from a null URI");
try
{
#if SILVERLIGHT
if (u.IsFile())
#else
if (u.IsFile)
#endif
{
//Invoke FileLoader instead
RaiseWarning("This is a file: URI so invoking the FileLoader instead");
if (Path.DirectorySeparatorChar == '/')
{
FileLoader.Load(handler, u.ToString().Substring(7), parser);
}
else
{
FileLoader.Load(handler, u.ToString().Substring(8), parser);
}
return;
}
if (u.Scheme.Equals("data"))
{
//Invoke DataUriLoader instead
RaiseWarning("This is a data: URI so invoking the DataUriLoader instead");
DataUriLoader.Load(handler, u);
return;
}
//Sanitise the URI to remove any Fragment ID
u = Tools.StripUriFragment(u);
#if !NO_URICACHE
//Use Cache if possible
String etag = String.Empty;
String local = null;
if (Options.UriLoaderCaching)
{
if (_cache.HasETag(u))
{
//Get the ETag and then we'll include an If-None-Match header in our request
etag = _cache.GetETag(u);
}
else if (_cache.HasLocalCopy(u, true))
{
//Just try loading from the local copy
local = _cache.GetLocalCopy(u);
if (local != null)
{
try
{
FileLoader.Load(handler, local, new TurtleParser());
}
catch
{
//If we get an Exception we failed to access the file successfully
_cache.RemoveETag(u);
_cache.RemoveLocalCopy(u);
UriLoader.Load(handler, u, parser);
}
return;
}
}
}
#endif
//Set-up the Request
HttpWebRequest httpRequest;
httpRequest = (HttpWebRequest)WebRequest.Create(u);
//Want to ask for RDF formats
if (parser != null)
{
//If a non-null parser set up a HTTP Header that is just for the given parser
httpRequest.Accept = MimeTypesHelper.CustomHttpAcceptHeader(parser);
}
else
{
httpRequest.Accept = MimeTypesHelper.HttpAcceptHeader;
}
#if !NO_URICACHE
if (Options.UriLoaderCaching)
{
if (!etag.Equals(String.Empty))
{
httpRequest.Headers.Add(HttpRequestHeader.IfNoneMatch, etag);
}
}
#endif
//Use HTTP GET
httpRequest.Method = "GET";
#if !SILVERLIGHT
httpRequest.Timeout = Options.UriLoaderTimeout;
#endif
if (_userAgent != null && !_userAgent.Equals(String.Empty))
{
httpRequest.UserAgent = _userAgent;
}
#if DEBUG
//HTTP Debugging
if (Options.HttpDebugging)
{
Tools.HttpDebugRequest(httpRequest);
}
#endif
using (HttpWebResponse httpResponse = (HttpWebResponse)httpRequest.GetResponse())
{
#if DEBUG
//HTTP Debugging
if (Options.HttpDebugging)
{
Tools.HttpDebugResponse(httpResponse);
}
#endif
#if !NO_URICACHE
if (Options.UriLoaderCaching)
{
//Are we using ETag based caching?
if (!etag.Equals(String.Empty))
{
//Did we get a Not-Modified response?
if (httpResponse.StatusCode == HttpStatusCode.NotModified)
{
//If so then we need to load the Local Copy assuming it exists?
if (_cache.HasLocalCopy(u, false))
{
local = _cache.GetLocalCopy(u);
try
{
FileLoader.Load(handler, local, new TurtleParser());
}
catch
{
//If we get an Exception we failed to access the file successfully
_cache.RemoveETag(u);
_cache.RemoveLocalCopy(u);
UriLoader.Load(handler, u, parser);
}
return;
}
else
{
//If the local copy didn't exist then we need to redo the response without
//the ETag as we've lost the cached copy somehow
_cache.RemoveETag(u);
UriLoader.Load(handler, u, parser);
return;
}
}
//If we didn't get a Not-Modified response then we'll continue and parse the new response
}
}
#endif
//Get a Parser and Load the RDF
if (parser == null)
{
//Only need to auto-detect the parser if a specific one wasn't specified
parser = MimeTypesHelper.GetParser(httpResponse.ContentType);
}
parser.Warning += RaiseWarning;
#if !NO_URICACHE
//To do caching we ask the cache to give us a handler and then we tie it to
IRdfHandler cacheHandler = _cache.ToCache(u, Tools.StripUriFragment(httpResponse.ResponseUri), httpResponse.Headers["ETag"]);
if (cacheHandler != null)
{
//Note: We can ONLY use caching when we know that the Handler will accept all the data returned
//i.e. if the Handler may trim the data in some way then we shouldn't cache the data returned
if (handler.AcceptsAll)
{
handler = new MultiHandler(new IRdfHandler[] { handler, cacheHandler });
}
else
{
cacheHandler = null;
}
}
try
{
#endif
parser.Load(handler, new StreamReader(httpResponse.GetResponseStream()));
#if !NO_URICACHE
}
catch
{
//If we were trying to cache the response and something went wrong discard the cached copy
_cache.RemoveETag(u);
_cache.RemoveETag(Tools.StripUriFragment(httpResponse.ResponseUri));
_cache.RemoveLocalCopy(u);
_cache.RemoveLocalCopy(Tools.StripUriFragment(httpResponse.ResponseUri));
}
#endif
}
}
catch (UriFormatException uriEx)
{
//Uri Format Invalid
throw new RdfParseException("Unable to load from the given URI '" + u.ToString() + "' since it's format was invalid", uriEx);
}
catch (WebException webEx)
{
#if DEBUG
if (webEx.Response != null && Options.HttpDebugging)
{
Tools.HttpDebugResponse((HttpWebResponse)webEx.Response);
}
#endif
#if !NO_URICACHE
if (webEx.Response != null)
{
if (((HttpWebResponse)webEx.Response).StatusCode == HttpStatusCode.NotModified)
{
//If so then we need to load the Local Copy assuming it exists?
if (_cache.HasLocalCopy(u, false))
{
String local = _cache.GetLocalCopy(u);
try
{
FileLoader.Load(handler, local, new TurtleParser());
}
catch
{
//If we get an Exception we failed to access the file successfully
_cache.RemoveETag(u);
_cache.RemoveLocalCopy(u);
UriLoader.Load(handler, u, parser);
}
return;
}
else
{
//If the local copy didn't exist then we need to redo the response without
//the ETag as we've lost the cached copy somehow
_cache.RemoveETag(u);
UriLoader.Load(handler, u, parser);
return;
}
}
}
#endif
//Some sort of HTTP Error occurred
throw new WebException("A HTTP Error occurred resolving the URI '" + u.ToString() + "'", webEx);
}
}
public void ParsingMultiHandlerBadInstantiation2()
{
MultiHandler handler = new MultiHandler(null);
}
public void ParsingMultiHandlerGraphAndNull()
{
EnsureTestData();
Graph g = new Graph();
GraphHandler handler1 = new GraphHandler(g);
NullHandler handler2 = new NullHandler();
MultiHandler handler = new MultiHandler(new IRdfHandler[] { handler1, handler2 });
TurtleParser parser = new TurtleParser();
parser.Load(handler, "temp.ttl");
}
public void ParsingMultiHandlerBadInstantiation3()
{
GraphHandler h = new GraphHandler(new Graph());
MultiHandler handler = new MultiHandler(new IRdfHandler[] { h, h });
}
public void ParsingMultiHandlerBadInstantiation()
{
MultiHandler handler = new MultiHandler(Enumerable.Empty <IRdfHandler>());
}
public void ParsingMultiHandlerBadInstantiation3()
{
GraphHandler h = new GraphHandler(new Graph());
MultiHandler handler = new MultiHandler(new IRdfHandler[] { h, h });
}
public void ParsingMultiHandlerGraphAndCount()
{
EnsureTestData();
Graph g = new Graph();
GraphHandler handler1 = new GraphHandler(g);
CountHandler handler2 = new CountHandler();
MultiHandler handler = new MultiHandler(new IRdfHandler[] { handler1, handler2 });
TurtleParser parser = new TurtleParser();
parser.Load(handler, "temp.ttl");
Assert.AreEqual(g.Triples.Count, handler2.Count, "Expected Counts to be the same");
}