public EntityCache(TimeSpan cacheExpiration) { _masterGraph = new JsonLdGraph(); _pages = new ConcurrentDictionary <Uri, JsonLdPage>(); _cacheExpiration = cacheExpiration; _tidyTimer = new System.Threading.Timer(TidyTimerTick, null, _cacheExpiration, _cacheExpiration); }
/// <summary> /// Add a graph to the master graph. /// </summary> private void MergeGraph(JsonLdGraph graph) { lock (this) { _masterGraph.Merge(graph); } }
public void Merge(JsonLdGraph graph) { lock (this) { foreach (var triple in graph.Triples) { AssertNoLock(triple); } } }
/// <summary> /// Removes all pages not used within the given time span. /// </summary> private void CleanUp(TimeSpan keepPagesUsedWithin) { DateTime cutOff = DateTime.UtcNow.Subtract(keepPagesUsedWithin); // lock to keep any new pages from being added during this lock (this) { // just in case we show really late if (_disposed) { return; } // create a working set of pages that can be considered locked JsonLdPage[] pages = _pages.Values.ToArray(); // if pages are still loading we should skip the clean up // TODO: post-preview this should force a clean up if the graph is huge if (pages.All(p => p.IsLoaded)) { // check if a clean up is needed if (pages.Any(p => !p.UsedAfter(cutOff))) { List <JsonLdPage> keep = new List <JsonLdPage>(pages.Length); List <JsonLdPage> remove = new List <JsonLdPage>(pages.Length); // pages could potentially change last accessed times, so make the decisions in one shot foreach (var page in pages) { if (page.UsedAfter(cutOff)) { keep.Add(page); } else { remove.Add(page); } } // second check to make sure we need to do this if (remove.Count > 0) { DataTraceSources.Verbose("[EntityCache] EntityCache rebuild started."); JsonLdGraph graph = new JsonLdGraph(); // graph merge foreach (var page in keep) { graph.Merge(page.Graph); } _masterGraph = graph; DataTraceSources.Verbose("[EntityCache] EntityCache rebuild complete."); // remove and dispose of the old pages foreach (var page in remove) { JsonLdPage removedPage = null; if (_pages.TryRemove(page.Uri, out removedPage)) { Debug.Assert(!removedPage.UsedAfter(cutOff), "Someone used a page that was scheduled to be removed. This should have been locked."); removedPage.Dispose(); } else { Debug.Fail(page.Uri.AbsoluteUri + " disappeared from the page cache."); } } } } } } }
/// <summary> /// Load a compacted json object into a JsonLdGraph /// </summary> public static JsonLdGraph Load(JObject compacted, JsonLdPage page) { Dictionary <int, JObject> nodes = new Dictionary <int, JObject>(); int marker = 0; // Mark each node with a serial number Action <JObject> addSerial = (node) => { if (!Utility.IsInContext(node)) { int serial = marker++; node[Constants.CacheNode] = serial; nodes.Add(serial, node); } }; // add serials Utility.JsonEntityVisitor(compacted, addSerial); // create graph without JTokens var basicGraph = Utility.GetGraphFromCompacted(compacted); // split out the cache triples List <Triple> normalTriples = new List <Triple>(); Dictionary <string, JObject> cacheTriples = new Dictionary <string, JObject>(); foreach (var triple in basicGraph.Triples) { // cache node predicates represent the mapping between the subject and token serial if (triple.Predicate.IsValue(Constants.CacheNode)) { string subject = triple.Subject.GetValue(); int serial; Int32.TryParse(triple.Object.GetValue(), out serial); // Remove the serial we added JObject jObject = nodes[serial]; jObject.Remove(Constants.CacheNode); // there should not be any duplicates here cacheTriples.Add(subject, jObject); } else { // store this to go into the graph normalTriples.Add(triple); } } // create the real graph JsonLdGraph jsonGraph = new JsonLdGraph(); // merge the graph data with the compacted json tokens foreach (var triple in normalTriples) { string subject = triple.Subject.GetValue(); JObject jObject = null; cacheTriples.TryGetValue(subject, out jObject); var jsonTriple = new JsonLdTriple(page, jObject, triple.Subject, triple.Predicate, triple.Object); jsonGraph.Assert(jsonTriple); } return(jsonGraph); }
private void Load(Action <JsonLdPage> callback) { JObject workingCopy = _compacted; try { if (!Utility.IsValidJsonLd(workingCopy)) { DataTraceSources.Verbose("[EntityCache] Invalid JsonLd skipping {0}", Uri.AbsoluteUri); // we can't parse this page, load a blank graph _graph = new JsonLdGraph(); return; } // we have to modify the json to create the graph, since other people are free to use // _compacted during this time we have to make a copy, after we finish we will throw // away _compacted and provide the copy we used instead. workingCopy = _compacted.DeepClone() as JObject; Uri rootUri = Utility.GetEntityUri(workingCopy); if (rootUri == null) { // remove the blank node string blankUrl = "http://blanknode.nuget.org/" + Guid.NewGuid().ToString(); workingCopy["@id"] = blankUrl; DataTraceSources.Verbose("[EntityCache] BlankNode Doc {0}", blankUrl); } DataTraceSources.Verbose("[EntityCache] Added {0}", Uri.AbsoluteUri); // Load _graph = JsonLdGraph.Load(workingCopy, this); // make the callback which should merge us into the master graph callback(this); } catch (Exception ex) { // Something horrible happened when parsing the json-ld. // The original file may be corrupted. The best option here is to leave the page // out of the entity cache. Requests for entities from the page should default to just returning // the compacted JTokens back. Those jtokens have as much info as we can get in this bad state. DataTraceSources.Verbose("[EntityCache] Unable to load!! {0} {1}", Uri.AbsoluteUri, ex.ToString()); } finally { if (_graph == null) { _graph = new JsonLdGraph(); } // replace the original with the copy we used for the graph _compacted = workingCopy; _loadWait.Set(); _isLoaded = true; } }