private static void DeleteCsvAndMetadata(string baseDirectory, string datasetId, IProgressLog progressLog)
{
Log.Information("DeleteCsvAndMetadata: {baseDirectory}, {datasetId}", baseDirectory, datasetId);
try
{
progressLog.Info("Deleting source CSV and CSV metadata files");
var csvPath = Path.Combine(baseDirectory, "csv", datasetId);
Directory.Delete(csvPath, true);
}
catch (Exception ex)
{
progressLog.Exception(ex, "Error deleting source CSV and CSV metadata files");
throw;
}
}
public bool HandleTripleCollection(IList <Triple> tripleCollection)
{
var subject = (tripleCollection[0].Subject as IUriNode)?.Uri;
try
{
if (_resourceMap.CanMap(subject))
{
if (_noFilter || tripleCollection.Any(t => _graphFilter.Contains(t.GraphUri)))
{
// There is some data in the graph that was updated, so we should regenerate the files for this resource
var targetPath = _resourceMap.GetPathFor(subject);
var targetDir = Path.GetDirectoryName(targetPath);
if (targetDir != null && !Directory.Exists(targetDir))
{
Directory.CreateDirectory(targetDir);
}
var graph = new Graph();
graph.Assert(tripleCollection);
#if GENERATE_RDFXML
GenerateRdfXmlFile(subject, targetPath, graph);
#endif
GenerateNQuadsFile(subject, targetPath, graph);
}
}
_subjectCount++;
if (_subjectCount % _reportInterval == 0)
{
_progressLog.Info("Generating static files - {0} subjects processed.", _subjectCount);
}
}
catch (Exception ex)
{
_progressLog.Exception(ex, "Error generating static files for subject {0}: {1}", subject, ex.Message);
}
return(true);
}
public bool HandleResource(INode resourceNode, IList <Triple> subjectStatements, IList <Triple> objectStatements)
{
if (!(resourceNode is IUriNode))
{
return(true);
}
if ((subjectStatements == null || subjectStatements.Count == 0) && (objectStatements == null || objectStatements.Count == 0))
{
return(true);
}
var subject = (resourceNode as IUriNode)?.Uri;
var nquads = subject == null ? null : _uriService.GetSubjectDataUrl(subject.ToString(), "nq");
try
{
var targetPath = _resourceMap.GetPathFor(subject);
if (targetPath != null)
{
targetPath += ".html";
var targetDir = Path.GetDirectoryName(targetPath);
if (!Directory.Exists(targetDir))
{
Directory.CreateDirectory(targetDir);
}
var parentDataset = string.Format("{0}://{1}", subject.Scheme, subject.Authority);
if (subject.ToString().IndexOf("id/resource", StringComparison.InvariantCultureIgnoreCase) > 0)
{
var datasetSegments = subject.Segments.Take(subject.Segments.Length - 1).ToArray();
foreach (var segment in datasetSegments)
{
if (segment.Equals("resource/"))
{
parentDataset += "dataset/";
}
else
{
parentDataset += segment;
}
}
parentDataset = parentDataset.Trim("/".ToCharArray());
}
UpdateVariables(nquads, parentDataset);
var html = _viewEngine.Render(subject, subjectStatements, objectStatements, _addVariables);
using (var stream = File.Open(targetPath, FileMode.Create, FileAccess.ReadWrite))
{
using (var writer = new StreamWriter(stream, Encoding.UTF8))
{
writer.Write(html);
}
stream.Close();
}
_numFilesGenerated++;
if (_numFilesGenerated % _reportInterval == 0)
{
_progressLog.Info("Generating static HTML files - {0} files created/updated.", _numFilesGenerated);
}
}
}
catch (Exception ex)
{
_progressLog.Exception(ex, "Error generating HTML file for subject {0}: {1}", subject, ex.Message);
}
return(true);
}
/// <summary>
/// Adds content to a dataset graph in the DataDock repository, optionally overwriting existing dataset content
/// </summary>
/// <param name="insertTriples"></param>
/// <param name="datasetIri"></param>
/// <param name="dropExistingGraph"></param>
/// <param name="metadataGraph"></param>
/// <param name="metadataGraphIri"></param>
/// <param name="definitionsGraph"></param>
/// <param name="definitionsGraphIri"></param>
/// <param name="publisherIri"></param>
/// <param name="publisherInfo"></param>
/// <param name="repositoryTitle"></param>
/// <param name="repositoryDescription"></param>
/// <param name="rootMetadataGraphIri"></param>
public void UpdateDataset(
IGraph insertTriples, Uri datasetIri, bool dropExistingGraph,
IGraph metadataGraph, Uri metadataGraphIri,
IGraph definitionsGraph, Uri definitionsGraphIri,
Uri publisherIri, ContactInfo publisherInfo,
string repositoryTitle, string repositoryDescription,
Uri rootMetadataGraphIri)
{
try
{
_progressLog.Info("Updating RDF repository");
if (dropExistingGraph)
{
// Drop dataset data graph
_progressLog.Info("Dropping all existing RDF data in graph {0}", datasetIri);
_quinceStore.DropGraph(datasetIri);
}
// Drop dataset metadata graph
_progressLog.Info("Dropping all metadata in graph {0}", metadataGraphIri);
_quinceStore.DropGraph(metadataGraphIri);
// Add triples to dataset data graph
_progressLog.Info("Adding new RDF data to graph {0} ({1} triples)", datasetIri, insertTriples.Triples.Count);
var addCount = 0;
foreach (var t in insertTriples.Triples)
{
_quinceStore.Assert(t.Subject, t.Predicate, t.Object, datasetIri);
addCount++;
if (addCount % 1000 == 0)
{
_progressLog.Info("Added {0} / {1} triples ({2}%)", addCount, insertTriples.Triples.Count,
addCount * 100 / insertTriples.Triples.Count);
}
}
// Add triples to dataset metadata graph
_progressLog.Info("Adding dataset metadata to graph {0} ({1} triples)", metadataGraphIri, metadataGraph.Triples.Count);
foreach (var t in metadataGraph.Triples)
{
_quinceStore.Assert(t.Subject, t.Predicate, t.Object, metadataGraphIri);
}
// Add triples to the definitions graph
_progressLog.Info("Adding column definition metadata to graph {0} ({1} triples)", definitionsGraphIri, definitionsGraph.Triples.Count);
foreach (var t in definitionsGraph.Triples)
{
_quinceStore.Assert(t.Subject, t.Predicate, t.Object, definitionsGraphIri);
}
// Update the root metadata graph to ensure it includes this dataset as a subset
_progressLog.Info("Updating root metadata");
var repositoryNode = insertTriples.CreateUriNode(_repositoryUri);
var subset = insertTriples.CreateUriNode(new Uri("http://rdfs.org/ns/void#subset"));
var rdfType = insertTriples.CreateUriNode(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"));
var rdfsLabel = insertTriples.CreateUriNode(new Uri("http://www.w3.org/2000/01/rdf-schema#label"));
var foafName = insertTriples.CreateUriNode(new Uri("http://xmlns.com/foaf/0.1/name"));
var foafMbox = insertTriples.CreateUriNode(new Uri("http://xmlns.com/foaf/0.1/mbox"));
var foafHomepage = insertTriples.CreateUriNode(new Uri("http://xmlns.com/foaf/0.1/homepage"));
var dctermsPublisher = insertTriples.CreateUriNode(new Uri("http://purl.org/dc/terms/publisher"));
var dctermsTitle = insertTriples.CreateUriNode(new Uri("http://purl.org/dc/terms/title"));
var dctermsDescription = insertTriples.CreateUriNode(new Uri("http://purl.org/dc/terms/description"));
var publisherNode = insertTriples.CreateUriNode(publisherIri);
_quinceStore.Assert(repositoryNode, rdfType, insertTriples.CreateUriNode(new Uri("http://rdfs.org/ns/void#Dataset")), rootMetadataGraphIri);
_quinceStore.Assert(repositoryNode, subset, insertTriples.CreateUriNode(datasetIri), rootMetadataGraphIri);
_quinceStore.Assert(repositoryNode, dctermsPublisher, publisherNode, rootMetadataGraphIri);
// Update repository title and description
foreach (var t in _quinceStore.GetTriplesForSubject(_repositoryUri))
{
if (t.GraphUri.Equals(rootMetadataGraphIri))
{
if (t.Predicate.Equals(dctermsTitle) || t.Predicate.Equals(dctermsDescription))
{
_quinceStore.Retract(t.Subject, t.Predicate, t.Object, rootMetadataGraphIri);
}
}
}
if (!string.IsNullOrEmpty(repositoryTitle))
{
_quinceStore.Assert(repositoryNode, dctermsTitle, insertTriples.CreateLiteralNode(repositoryTitle), rootMetadataGraphIri);
}
if (!string.IsNullOrEmpty(repositoryDescription))
{
_quinceStore.Assert(repositoryNode, dctermsDescription, insertTriples.CreateLiteralNode(repositoryDescription), rootMetadataGraphIri);
}
// Update publisher information
foreach (var t in _quinceStore.GetTriplesForSubject(publisherIri))
{
if (t.GraphUri.Equals(rootMetadataGraphIri))
{
_quinceStore.Retract(t.Subject, t.Predicate, t.Object, rootMetadataGraphIri);
}
}
if (!string.IsNullOrEmpty(publisherInfo?.Type))
{
var publisherType = insertTriples.CreateUriNode(new Uri(publisherInfo.Type));
_quinceStore.Assert(publisherNode, rdfType, publisherType, rootMetadataGraphIri);
}
if (!string.IsNullOrEmpty(publisherInfo?.Label))
{
var publisherLabel = insertTriples.CreateLiteralNode(publisherInfo.Label);
_quinceStore.Assert(publisherNode, rdfsLabel, publisherLabel, rootMetadataGraphIri);
_quinceStore.Assert(publisherNode, foafName, publisherLabel, rootMetadataGraphIri);
}
if (!string.IsNullOrEmpty(publisherInfo?.Email))
{
_quinceStore.Assert(publisherNode, foafMbox, insertTriples.CreateLiteralNode(publisherInfo.Email), rootMetadataGraphIri);
}
if (!string.IsNullOrEmpty(publisherInfo?.Website))
{
_quinceStore.Assert(publisherNode, foafHomepage, insertTriples.CreateUriNode(new Uri(publisherInfo.Website)), rootMetadataGraphIri);
}
// Flush all data to disk
_quinceStore.Flush();
}
catch (Exception ex)
{
Log.Error(ex, "CSV conversion failed");
throw new WorkerException(ex, "CSV conversion failed. Error updating RDF repository.");
}
}