public EntityCache(TimeSpan cacheExpiration)
{
_masterGraph = new JsonLdGraph();
_pages = new ConcurrentDictionary <Uri, JsonLdPage>();
_cacheExpiration = cacheExpiration;
_tidyTimer = new System.Threading.Timer(TidyTimerTick, null, _cacheExpiration, _cacheExpiration);
}
/// <summary>
/// Add a graph to the master graph.
/// </summary>
private void MergeGraph(JsonLdGraph graph)
{
lock (this)
{
_masterGraph.Merge(graph);
}
}
public void Merge(JsonLdGraph graph)
{
lock (this)
{
foreach (var triple in graph.Triples)
{
AssertNoLock(triple);
}
}
}
/// <summary>
/// Removes all pages not used within the given time span.
/// </summary>
private void CleanUp(TimeSpan keepPagesUsedWithin)
{
DateTime cutOff = DateTime.UtcNow.Subtract(keepPagesUsedWithin);
// lock to keep any new pages from being added during this
lock (this)
{
// just in case we show really late
if (_disposed)
{
return;
}
// create a working set of pages that can be considered locked
JsonLdPage[] pages = _pages.Values.ToArray();
// if pages are still loading we should skip the clean up
// TODO: post-preview this should force a clean up if the graph is huge
if (pages.All(p => p.IsLoaded))
{
// check if a clean up is needed
if (pages.Any(p => !p.UsedAfter(cutOff)))
{
List <JsonLdPage> keep = new List <JsonLdPage>(pages.Length);
List <JsonLdPage> remove = new List <JsonLdPage>(pages.Length);
// pages could potentially change last accessed times, so make the decisions in one shot
foreach (var page in pages)
{
if (page.UsedAfter(cutOff))
{
keep.Add(page);
}
else
{
remove.Add(page);
}
}
// second check to make sure we need to do this
if (remove.Count > 0)
{
DataTraceSources.Verbose("[EntityCache] EntityCache rebuild started.");
JsonLdGraph graph = new JsonLdGraph();
// graph merge
foreach (var page in keep)
{
graph.Merge(page.Graph);
}
_masterGraph = graph;
DataTraceSources.Verbose("[EntityCache] EntityCache rebuild complete.");
// remove and dispose of the old pages
foreach (var page in remove)
{
JsonLdPage removedPage = null;
if (_pages.TryRemove(page.Uri, out removedPage))
{
Debug.Assert(!removedPage.UsedAfter(cutOff), "Someone used a page that was scheduled to be removed. This should have been locked.");
removedPage.Dispose();
}
else
{
Debug.Fail(page.Uri.AbsoluteUri + " disappeared from the page cache.");
}
}
}
}
}
}
}
/// <summary>
/// Load a compacted json object into a JsonLdGraph
/// </summary>
public static JsonLdGraph Load(JObject compacted, JsonLdPage page)
{
Dictionary <int, JObject> nodes = new Dictionary <int, JObject>();
int marker = 0;
// Mark each node with a serial number
Action <JObject> addSerial = (node) =>
{
if (!Utility.IsInContext(node))
{
int serial = marker++;
node[Constants.CacheNode] = serial;
nodes.Add(serial, node);
}
};
// add serials
Utility.JsonEntityVisitor(compacted, addSerial);
// create graph without JTokens
var basicGraph = Utility.GetGraphFromCompacted(compacted);
// split out the cache triples
List <Triple> normalTriples = new List <Triple>();
Dictionary <string, JObject> cacheTriples = new Dictionary <string, JObject>();
foreach (var triple in basicGraph.Triples)
{
// cache node predicates represent the mapping between the subject and token serial
if (triple.Predicate.IsValue(Constants.CacheNode))
{
string subject = triple.Subject.GetValue();
int serial;
Int32.TryParse(triple.Object.GetValue(), out serial);
// Remove the serial we added
JObject jObject = nodes[serial];
jObject.Remove(Constants.CacheNode);
// there should not be any duplicates here
cacheTriples.Add(subject, jObject);
}
else
{
// store this to go into the graph
normalTriples.Add(triple);
}
}
// create the real graph
JsonLdGraph jsonGraph = new JsonLdGraph();
// merge the graph data with the compacted json tokens
foreach (var triple in normalTriples)
{
string subject = triple.Subject.GetValue();
JObject jObject = null;
cacheTriples.TryGetValue(subject, out jObject);
var jsonTriple = new JsonLdTriple(page, jObject, triple.Subject, triple.Predicate, triple.Object);
jsonGraph.Assert(jsonTriple);
}
return(jsonGraph);
}
private void Load(Action <JsonLdPage> callback)
{
JObject workingCopy = _compacted;
try
{
if (!Utility.IsValidJsonLd(workingCopy))
{
DataTraceSources.Verbose("[EntityCache] Invalid JsonLd skipping {0}", Uri.AbsoluteUri);
// we can't parse this page, load a blank graph
_graph = new JsonLdGraph();
return;
}
// we have to modify the json to create the graph, since other people are free to use
// _compacted during this time we have to make a copy, after we finish we will throw
// away _compacted and provide the copy we used instead.
workingCopy = _compacted.DeepClone() as JObject;
Uri rootUri = Utility.GetEntityUri(workingCopy);
if (rootUri == null)
{
// remove the blank node
string blankUrl = "http://blanknode.nuget.org/" + Guid.NewGuid().ToString();
workingCopy["@id"] = blankUrl;
DataTraceSources.Verbose("[EntityCache] BlankNode Doc {0}", blankUrl);
}
DataTraceSources.Verbose("[EntityCache] Added {0}", Uri.AbsoluteUri);
// Load
_graph = JsonLdGraph.Load(workingCopy, this);
// make the callback which should merge us into the master graph
callback(this);
}
catch (Exception ex)
{
// Something horrible happened when parsing the json-ld.
// The original file may be corrupted. The best option here is to leave the page
// out of the entity cache. Requests for entities from the page should default to just returning
// the compacted JTokens back. Those jtokens have as much info as we can get in this bad state.
DataTraceSources.Verbose("[EntityCache] Unable to load!! {0} {1}", Uri.AbsoluteUri, ex.ToString());
}
finally
{
if (_graph == null)
{
_graph = new JsonLdGraph();
}
// replace the original with the copy we used for the graph
_compacted = workingCopy;
_loadWait.Set();
_isLoaded = true;
}
}
