public void ParsingSpeedNTriples1Million()
{
try
{
Options.InternUris = false;
EnsureTestData(1000000, "million.nt", new NTriplesFormatter());
Graph g = new Graph();
Stopwatch watch = new Stopwatch();
NTriplesParser parser = new NTriplesParser();
watch.Start();
parser.Load(g, "million.nt");
watch.Stop();
Console.WriteLine(watch.Elapsed.ToString());
this.CalculateSpeed(1000000, watch);
}
catch (OutOfMemoryException)
{
throw new SkipTestException("Out of memory when parsing 1000000 NTriples");
}
finally
{
Options.InternUris = true;
}
}
public void ParsingSpeedNTriples500Thousand()
{
try
{
Options.InternUris = false;
EnsureTestData(500000, "500thou.nt", new NTriplesFormatter());
Graph g = new Graph();
Stopwatch watch = new Stopwatch();
NTriplesParser parser = new NTriplesParser();
watch.Start();
parser.Load(g, "500thou.nt");
watch.Stop();
Console.WriteLine(watch.Elapsed.ToString());
this.CalculateSpeed(500000, watch);
}
finally
{
Options.InternUris = true;
}
}
public override Stream GetFragment(string id, string contentType)
{
// need to see which definition we match
ResourcePublishingDefinition definition = null;
UriTemplateMatch match = null;
foreach (var resourcePublishingDefinition in _publishingDefinitions)
{
var newuri = new Uri(id);
match = resourcePublishingDefinition.UriTemplate.Match(resourcePublishingDefinition.ResourcePrefix, newuri);
if (match != null)
{
definition = resourcePublishingDefinition;
break;
}
}
if (definition == null) { throw new Exception("Unable to find matching definition for uri " + id); }
var sb = new StringBuilder();
foreach (var generationDefinition in definition.FragmentGenerationDefinitions)
{
try
{
var data = ExecuteQuery(_dataSourceConnectionString, generationDefinition.FragmentQuery.Replace("[[id]]", match.BoundVariables["id"]));
foreach (DataRow row in data.Rows)
{
var dra = new DbDataRow(row);
foreach (var line in generationDefinition.RdfTemplateLines)
{
var linePattern = new NTripleLinePattern(line);
linePattern.GenerateNTriples(sb, dra, generationDefinition.GenericTemplateExcludeColumns, contentType.Equals("xml"));
}
}
}
catch (Exception ex)
{
Logging.LogError(1, "Error processing definition {0} {1} {2} {3}", ex.Message, generationDefinition.SnapshotQuery, _dataSourceConnectionString, ex.StackTrace);
}
}
try
{
var g = new Graph();
var parser = new NTriplesParser(TokenQueueMode.SynchronousBufferDuringParsing);
var triplesStr = sb.ToString();
parser.Load(g, new StringReader(triplesStr));
if (contentType.Equals("xml"))
{
var ms = new MemoryStream();
var sw = new StreamWriter(ms, Encoding.UTF8);
var rdfxmlwriter = new RdfXmlWriter();
var strw = new System.IO.StringWriter();
rdfxmlwriter.Save(g, strw);
var data = strw.ToString();
data = data.Replace("~~~2B~~~", "%2B");
data = data.Replace("~~~SLASH~~~", "%2F");
data = data.Replace("utf-16", "utf-8");
sw.Write(data);
sw.Flush();
ms.Seek(0, SeekOrigin.Begin);
return ms;
}
else
{
var ms = new MemoryStream();
var sw = new StreamWriter(ms);
sw.Write(triplesStr);
sw.Flush();
ms.Seek(0, SeekOrigin.Begin);
return ms;
}
}
catch (Exception ex)
{
Logging.LogError(1, "Error getting fragment {0} {1}", ex.Message, ex.StackTrace);
throw;
}
}
public void ParsingSpeedNTriples1Million()
{
EnsureTestData(1000000, "million.nt", new NTriplesFormatter());
Graph g = new Graph(new IndexedTripleCollection(1));
Stopwatch watch = new Stopwatch();
NTriplesParser parser = new NTriplesParser();
watch.Start();
parser.Load(g, "million.nt");
watch.Stop();
Console.WriteLine(watch.Elapsed);
this.CalculateSpeed(1000000, watch);
}
public static Object LoadFromReader(Reader r, string baseUri, org.openrdf.rio.RDFFormat rdff)
{
Object obj;
if (rdff == dotSesameFormats.RDFFormat.N3)
{
obj = new Graph();
if (baseUri != null) ((IGraph)obj).BaseUri = new Uri(baseUri);
Notation3Parser parser = new Notation3Parser();
parser.Load((IGraph)obj, r.ToDotNetReadableStream());
}
else if (rdff == dotSesameFormats.RDFFormat.NTRIPLES)
{
obj = new Graph();
if (baseUri != null) ((IGraph)obj).BaseUri = new Uri(baseUri);
NTriplesParser parser = new NTriplesParser();
parser.Load((IGraph)obj, r.ToDotNetReadableStream());
}
else if (rdff == dotSesameFormats.RDFFormat.RDFXML)
{
obj = new Graph();
if (baseUri != null) ((IGraph)obj).BaseUri = new Uri(baseUri);
RdfXmlParser parser = new RdfXmlParser();
parser.Load((IGraph)obj, r.ToDotNetReadableStream());
}
else if (rdff == dotSesameFormats.RDFFormat.TRIG)
{
obj = new TripleStore();
TriGParser trig = new TriGParser();
trig.Load((ITripleStore)obj, new StreamParams(r.ToDotNetReadableStream().BaseStream));
}
else if (rdff == dotSesameFormats.RDFFormat.TRIX)
{
obj = new TripleStore();
TriXParser trix = new TriXParser();
trix.Load((ITripleStore)obj, new StreamParams(r.ToDotNetReadableStream().BaseStream));
}
else if (rdff == dotSesameFormats.RDFFormat.TURTLE)
{
obj = new Graph();
if (baseUri != null) ((IGraph)obj).BaseUri = new Uri(baseUri);
TurtleParser parser = new TurtleParser();
parser.Load((IGraph)obj, r.ToDotNetReadableStream());
}
else
{
throw new RdfParserSelectionException("The given Input Format is not supported by dotNetRDF");
}
return obj;
}
public static void Main(string[] args)
{
StreamWriter output = new StreamWriter("NTriplesTestSuite.txt");
String[] wantTrace = { "test.nt" };
String[] wantOutput = { };
bool outputAll = true;
String[] skipTests = { };
Console.SetOut(output);
try
{
int testsPassed = 0;
int testsFailed = 0;
String[] files = Directory.GetFiles("ntriples_tests");
NTriplesParser parser = new NTriplesParser();
bool passed, passDesired;
Graph g = new Graph();
foreach (String file in files)
{
if (skipTests.Contains(Path.GetFileName(file)))
{
output.WriteLine("## Skipping Test of File " + Path.GetFileName(file));
output.WriteLine();
continue;
}
if (Path.GetExtension(file) != ".nt")
{
continue;
}
Debug.WriteLine("Testing File " + Path.GetFileName(file));
output.WriteLine("## Testing File " + Path.GetFileName(file));
output.WriteLine("# Test Started at " + DateTime.Now.ToString(TestSuite.TestSuiteTimeFormat));
passed = false;
passDesired = true;
try
{
g = new Graph();
g.BaseUri = new Uri(TestSuiteBaseUri);
if (Path.GetFileNameWithoutExtension(file).StartsWith("bad"))
{
passDesired = false;
output.WriteLine("# Desired Result = Parsing Failed");
}
else
{
output.WriteLine("# Desired Result = Parses OK");
}
if (wantTrace.Contains(Path.GetFileName(file)))
{
parser.TraceTokeniser = true;
}
else
{
parser.TraceTokeniser = false;
}
parser.Load(g, file);
passed = true;
output.WriteLine("Parsed OK");
if (outputAll || wantOutput.Contains(Path.GetFileName(file)))
{
NTriplesWriter writer = new NTriplesWriter();
writer.Save(g, "ntriples_tests\\" + Path.GetFileNameWithoutExtension(file) + ".out");
}
}
catch (IOException ioEx)
{
reportError(output, "IO Exception", ioEx);
}
catch (RdfParseException parseEx)
{
reportError(output, "Parsing Exception", parseEx);
}
catch (RdfException rdfEx)
{
reportError(output, "RDF Exception", rdfEx);
}
catch (Exception ex)
{
reportError(output, "Other Exception", ex);
}
finally
{
if (passed && passDesired)
{
//Passed and we wanted to Pass
testsPassed++;
output.WriteLine("# Result = Test Passed");
}
else if (!passed && passDesired)
{
//Failed when we should have Passed
testsFailed++;
output.WriteLine("# Result = Test Failed");
}
else if (passed && !passDesired)
{
//Passed when we should have Failed
testsFailed++;
output.WriteLine("# Result = Test Failed");
}
else
{
//Failed and we wanted to Fail
testsPassed++;
output.WriteLine("# Result = Test Passed");
}
output.WriteLine("# Triples Generated = " + g.Triples.Count());
output.WriteLine("# Test Ended at " + DateTime.Now.ToString(TestSuite.TestSuiteTimeFormat));
}
output.WriteLine();
}
output.WriteLine(testsPassed + " Tests Passed");
output.WriteLine(testsFailed + " Tests Failed");
}
catch (Exception ex)
{
reportError(output, "Other Exception", ex);
}
Console.SetOut(Console.Out);
Console.WriteLine("Done");
Debug.WriteLine("Finished");
output.Close();
}
