/// <summary>
/// Commits all changed and new pages to the page store
/// </summary>
/// <param name="commitId">The transaction identifier for the commit</param>
/// <param name="profiler"></param>
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Commit"))
{
using (var outputStream = _persistenceManager.GetOutputStream(_filePath, FileMode.Open))
{
try
{
foreach (var entry in _modifiedPages.OrderBy(e => e.Key))
{
entry.Value.Write(outputStream, commitId);
lock (_pageCacheLock)
{
PageCache.Instance.InsertOrUpdate(_filePath, entry.Value);
}
}
_modifiedPages.Clear();
}
catch (Exception)
{
_modifiedPages.Clear();
throw;
}
}
CurrentTransactionId = commitId;
}
}
public void Insert(ulong transactionId, string resource, out ulong pageId, out byte segmentId, BrightstarProfiler profiler)
{
using (profiler.Step("ResourceTable.Insert"))
{
var byteCount = Encoding.UTF8.GetByteCount(resource);
var resourceBytes = new byte[byteCount + 4];
BitConverter.GetBytes(byteCount).CopyTo(resourceBytes, 0);
Encoding.UTF8.GetBytes(resource, 0, resource.Length, resourceBytes, 4);
lock (_writeLock)
{
if (_nextSegment == _pointerSegment)
{
StartNewPage(transactionId, profiler);
}
pageId = _currentPage;
segmentId = _nextSegment;
for (int i = 0; i < (byteCount + 4); i += _segmentSize)
{
_pageStore.Write(transactionId, _currentPage, resourceBytes, i, _nextSegment*_segmentSize,
_segmentSize < (byteCount + 4 - i) ? _segmentSize : (byteCount + 4 - i),
profiler);
_nextSegment++;
if (_nextSegment == _pointerSegment)
{
StartNewPage(transactionId, profiler);
}
}
}
}
}
private IResource CreateLongUriResource(ulong txnId, string uri, BrightstarProfiler profiler)
{
ulong pageId;
byte segId;
_resourceTable.Insert(txnId, uri, out pageId, out segId, profiler);
return new LongUriResource(uri, pageId, segId);
}
public INode GetNode(ulong nodeId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.GetNode"))
{
INode ret;
if (_nodeCache.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return ret;
}
profiler.Incr("NodeCache Miss");
using (profiler.Step("Load Node"))
{
var nodePage = _pageStore.Retrieve(nodeId, profiler);
var header = BitConverter.ToInt32(nodePage.Data, 0);
if (header < 0)
{
ret = MakeInternalNode(nodePage, ~header);
#if DEBUG_BTREE
_config.BTreeDebug("{0}: Loaded INTERNAL node from page {1}. {2}",_config.DebugId, nodePage.Id, ret.ToString());
#endif
}
else
{
ret = MakeLeafNode(nodePage, header);
#if DEBUG_BTREE
_config.BTreeDebug("{0}: Loaded LEAF node from page {1}. {2}", _config.DebugId, nodePage.Id, ret.ToString());
#endif
}
_nodeCache.Add(ret);
return ret;
}
}
}
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Commit"))
{
var pagesToWrite = _newPages.Where(rf => rf.IsAlive).Select(rf => rf.Target as IPage).Where(pg => pg != null);
if (_backgroundPageWriter != null)
{
foreach (var page in pagesToWrite)
{
_backgroundPageWriter.QueueWrite(page, commitId);
}
_backgroundPageWriter.Flush();
RestartBackgroundWriter();
}
else
{
using (var outputStream = _peristenceManager.GetOutputStream(_path, FileMode.Open))
{
foreach (var page in pagesToWrite)
{
page.Write(outputStream, commitId);
}
}
}
_newPages.Clear();
_newPageOffset = _nextPageId;
PageCache.Instance.Clear(_path);
}
}
public void TestSimpleJoinQuery()
{
var sparqlQuery =
@"PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/>
SELECT ?review WHERE {
?review bsbm:reviewFor ?product .
?product bsbm:productFeature <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature2330> .
} LIMIT 3";
// Warm-up
var client = BrightstarService.GetClient("type=embedded;storesDirectory=stores");
for (int i = 0; i < 5; i++)
{
client.ExecuteQuery("SparqlPerformance", sparqlQuery);
}
// Profile
var profiler = new BrightstarProfiler("SimpleJoinQuery");
for (int i = 0; i < 1000; i++)
{
using (profiler.Step("ExecuteQuery"))
{
client.ExecuteQuery("SparqlPerformance", sparqlQuery);
}
}
Console.WriteLine(profiler.GetLogString());
}
/// <summary>
/// Retrieves the resource with the specified resource ID
/// </summary>
/// <param name="resourceId">The resource ID to look for</param>
/// <param name="addToCache">Boolean flag indicating if the returned resource structure should be cached for faster future lookups</param>
/// <param name="profiler">OPTIONAL: Profiler to use to profile this call</param>
/// <returns>The corresponding resource or null if no match is found</returns>
public IResource GetResource(ulong resourceId, bool addToCache, BrightstarProfiler profiler = null)
{
using (profiler.Step("ResourceIndex.GetResource"))
{
IResource resource;
if (_resourceCache.TryGetValue(resourceId, out resource))
{
return(resource);
}
var buff = new byte[64];
if (Search(resourceId, buff, profiler))
{
resource = _resourceStore.FromBTreeValue(buff);
_resourceCache.Add(resourceId, resource);
return(resource);
}
#if DEBUG
if (resourceId > 0)
{
// Repeat the search for debug purposes
Search(resourceId, buff, profiler);
}
#endif
return(null);
}
}
/// <summary>
/// Opens an existing tree in the page store
/// </summary>
/// <param name="pageStore"></param>
/// <param name="rootPageId">The page ID of the BTree root node</param>
/// <param name="keySize"></param>
/// <param name="dataSize"></param>
/// <param name="profiler"></param>
public BPlusTree(IPageStore pageStore, ulong rootPageId, int keySize = 8, int dataSize = 64, BrightstarProfiler profiler = null)
{
_config = new BPlusTreeConfiguration(pageStore, keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
var root = GetNode(rootPageId, profiler);
_rootId = root.PageId;
}
private IEnumerable<KeyValuePair<byte[], ulong>>MakeInternalNodes(ulong txnId, IEnumerable<KeyValuePair<byte[], ulong >> children, BrightstarProfiler profiler)
{
var enumerator = children.GetEnumerator();
var childList = enumerator.Next(_internalBranchFactor).ToList();
if (childList.Count == 1)
{
yield return childList[0];
yield break;
}
byte[] prevNodeKey = childList[0].Key;
IInternalNode prevNode = MakeInternalNode(txnId, childList);
childList = enumerator.Next(_internalBranchFactor).ToList();
while(childList.Count > 0)
{
IInternalNode nextNode = MakeInternalNode(txnId, childList);
var nextNodeKey = childList[0].Key;
if (nextNode.NeedJoin)
{
nextNodeKey = new byte[_config.KeySize];
nextNode.RedistributeFromLeft(txnId, prevNode, childList[0].Key, nextNodeKey);
}
yield return WriteNode(txnId, prevNode, prevNodeKey, profiler);
prevNode = nextNode;
prevNodeKey = nextNodeKey;
childList = enumerator.Next(_internalBranchFactor).ToList();
}
yield return WriteNode(txnId, prevNode, prevNodeKey, profiler);
}
public int Preload(int maxPages, BrightstarProfiler profiler = null)
{
int pagesLoaded = this.PreloadTree(maxPages, profiler);
int remainingPages = maxPages - pagesLoaded;
if (remainingPages > 0)
{
// We have managed to load the complete index of predicate trees into memory
// See if there is enough room to preload at least the root node of each predicate tree
var numPredicatesToLoad = EnumeratePredicates(profiler).Count();
if (remainingPages >= numPredicatesToLoad)
{
// We are OK to load some pages from each predicate.
// It would be relatively expensive to precompute which indexes have most pages
// instead we will just enumerate through them loading remainingPages / numPredicatesToLoad
foreach (var predicateIndex in EnumeratePredicateIndexes(profiler))
{
pagesLoaded += predicateIndex.Value.PreloadTree(remainingPages / numPredicatesToLoad, profiler);
remainingPages = maxPages - pagesLoaded;
if (remainingPages <= 0)
{
break;
}
numPredicatesToLoad--;
}
}
}
return(pagesLoaded);
}
private IResource CreateLongLiteralResource(ulong txnId, string resourceValue, ulong dataTypeId, ulong langCodeId, BrightstarProfiler profiler)
{
ulong pageId;
byte segId;
_resourceTable.Insert(txnId, resourceValue, out pageId, out segId, profiler);
return new LongLiteralResource(resourceValue, dataTypeId, langCodeId, pageId, segId);
}
public void Delete(ulong txnId, byte[] key, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.Delete"))
{
var root = GetNode(_rootId, profiler);
if (root is ILeafNode)
{
(root as ILeafNode).Delete(txnId, key);
MarkDirty(txnId, root, profiler);
// Update root page pointer - see note in Insert() method above
_rootId = root.PageId;
}
else
{
bool underAllocation;
Delete(txnId, root as IInternalNode, key, out underAllocation, profiler);
if (root.KeyCount == 0)
{
// Now has only a single child leaf node, which should become the new tree root
root = GetNode((root as IInternalNode).GetChildPointer(0), profiler);
_rootId = root.PageId;
}
else
{
// Update root page pointer - see note in Insert() method above
_rootId = root.PageId;
}
}
}
}
public INode GetNode(ulong nodeId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.GetNode"))
{
INode ret;
if (_nodeCache.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return(ret);
}
profiler.Incr("NodeCache Miss");
using (profiler.Step("Load Node"))
{
var nodePage = _pageStore.Retrieve(nodeId, profiler);
var header = BitConverter.ToInt32(nodePage.Data, 0);
if (header < 0)
{
ret = MakeInternalNode(nodePage, ~header);
#if DEBUG_BTREE
_config.BTreeDebug("{0}: Loaded INTERNAL node from page {1}. {2}", _config.DebugId, nodePage.Id, ret.ToString());
#endif
}
else
{
ret = MakeLeafNode(nodePage, header);
#if DEBUG_BTREE
_config.BTreeDebug("{0}: Loaded LEAF node from page {1}. {2}", _config.DebugId, nodePage.Id, ret.ToString());
#endif
}
_nodeCache.Add(ret);
return(ret);
}
}
}
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
if (CanWrite)
{
foreach (var pageId in _modifiedPages.Keys)
{
var page = PageCache.Instance.Lookup(_partitionId, pageId) as BinaryFilePage;
if (page != null && page.IsDirty)
{
_backgroundPageWriter.QueueWrite(page, commitId);
}
}
_backgroundPageWriter.Flush();
lock (_restartLock)
{
_backgroundPageWriter.Shutdown();
_backgroundPageWriter.Dispose();
PageCache.Instance.Clear(_partitionId);
UpdatePartitionId();
_readTxnId = _writeTxnId;
_writeTxnId++;
_backgroundPageWriter =
new BackgroundPageWriter(_persistenceManager.GetOutputStream(_filePath, FileMode.Open));
}
}
else
{
throw new InvalidOperationException("Attempt to Commit on a read-only store instance");
}
}
public IPage Retrieve(ulong pageId, BrightstarProfiler profiler)
{
using (profiler.Step("BinaryFilePageStore.GetPage"))
{
// Look in the page cache
IPage page = PageCache.Instance.Lookup(_partitionId, pageId) as BinaryFilePage;
if (page != null)
{
profiler.Incr("PageCache Hit");
return(page);
}
// See if the page is queued for writing
if (_backgroundPageWriter != null && _backgroundPageWriter.TryGetPage(pageId, out page))
{
profiler.Incr("BackgroundWriter Queue Hit");
return(page);
}
// Not found in memory, so go to the disk
profiler.Incr("PageCache Miss");
using (profiler.Step("Load Page"))
{
page = _modifiedPages.ContainsKey(pageId)
? new BinaryFilePage(_inputStream, pageId, _nominalPageSize, _writeTxnId, true)
: new BinaryFilePage(_inputStream, pageId, _nominalPageSize, _readTxnId, false);
PageCache.Instance.InsertOrUpdate(_partitionId, page);
return(page);
}
}
}
public PrefixManager(IPageStore pageStore, ulong startPageId, BrightstarProfiler profiler)
{
_pageStore = pageStore;
var startPage = _pageStore.Retrieve(startPageId, profiler);
Load(startPage, profiler);
}
public byte[] Retrieve(ulong pageId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Retrieve"))
{
if (!_readonly && pageId >= _newPageOffset)
{
var newPage = _newPages[(int)(pageId - _newPageOffset)];
return(newPage.Data);
}
var page = PageCache.Instance.Lookup(_path, pageId) as FilePage;
if (page != null)
{
profiler.Incr("PageCache Hit");
return(page.Data);
}
using (profiler.Step("Load Page"))
{
profiler.Incr("PageCache Miss");
using (profiler.Step("Create FilePage"))
{
// Lock on stream to prevent attempts to concurrently load a page
lock (_stream)
{
page = new FilePage(_stream, pageId, _pageSize);
}
}
using (profiler.Step("Add FilePage To Cache"))
{
PageCache.Instance.InsertOrUpdate(_path, page);
}
return(page.Data);
}
}
}
public byte[] Retrieve(ulong pageId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Retrieve"))
{
if (!_readonly && pageId >= _newPageOffset)
{
var newPage = _newPages[(int) (pageId - _newPageOffset)];
return newPage.Data;
}
var page = PageCache.Instance.Lookup(_path, pageId) as FilePage;
if (page != null)
{
profiler.Incr("PageCache Hit");
return page.Data;
}
using (profiler.Step("Load Page"))
{
profiler.Incr("PageCache Miss");
using (profiler.Step("Create FilePage"))
{
// Lock on stream to prevent attempts to concurrently load a page
lock (_stream)
{
page = new FilePage(_stream, pageId, _pageSize);
}
}
using (profiler.Step("Add FilePage To Cache"))
{
PageCache.Instance.InsertOrUpdate(_path, page);
}
return page.Data;
}
}
}
/// <summary>
/// Retrieve the resource at the specified page and segment offset
/// </summary>
/// <param name="pageId">The ID of the page that holds the resource to be retrieved</param>
/// <param name="segment">The index of the segment within the page that holds the start of the resource</param>
/// <param name="profiler"></param>
/// <returns>The resource</returns>
public string GetResource(ulong pageId, byte segment, BrightstarProfiler profiler)
{
using (profiler.Step("ResourceTable.GetResource"))
{
var currentPage = _pageStore.Retrieve(pageId, profiler);
int resourceLength = BitConverter.ToInt32(currentPage, segment*_segmentSize);
int totalLength = resourceLength + 4;
int segmentsToLoad = totalLength/_segmentSize;
if (totalLength%_segmentSize > 0) segmentsToLoad++;
var buffer = new byte[segmentsToLoad*_segmentSize];
byte segmentIndex = segment;
for (int i = 0; i < segmentsToLoad; i++)
{
if (segmentIndex == _pointerSegment)
{
ulong nextPageId = BitConverter.ToUInt64(currentPage, _pageStore.PageSize - 8);
currentPage = _pageStore.Retrieve(nextPageId, profiler);
segmentIndex = 0;
}
Array.Copy(currentPage, segmentIndex*_segmentSize, buffer, i*_segmentSize, _segmentSize);
segmentIndex++;
}
return Encoding.UTF8.GetString(buffer, 4, resourceLength);
}
}
public INode GetNode(ulong nodeId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.GetNode"))
{
INode ret;
if (_modifiedNodes.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return ret;
}
if (_nodeCache.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return ret;
}
profiler.Incr("NodeCache Miss");
using (profiler.Step("Load Node"))
{
var nodePage = _pageStore.Retrieve(nodeId, profiler);
var header = BitConverter.ToInt32(nodePage, 0);
if (header < 0)
{
ret = new InternalNode(nodeId, nodePage, ~header, _config);
}
else
{
ret = new LeafNode(nodeId, nodePage, header, _config);
}
_nodeCache.Add(ret);
return ret;
}
}
}
/// <summary>
/// Retrieve the resource at the specified page and segment offset
/// </summary>
/// <param name="pageId">The ID of the page that holds the resource to be retrieved</param>
/// <param name="segment">The index of the segment within the page that holds the start of the resource</param>
/// <param name="profiler"></param>
/// <returns>The resource</returns>
public string GetResource(ulong pageId, byte segment, BrightstarProfiler profiler)
{
using (profiler.Step("ResourceTable.GetResource"))
{
var currentPage = _pageStore.Retrieve(pageId, profiler);
int resourceLength = BitConverter.ToInt32(currentPage, segment * _segmentSize);
int totalLength = resourceLength + 4;
int segmentsToLoad = totalLength / _segmentSize;
if (totalLength % _segmentSize > 0)
{
segmentsToLoad++;
}
var buffer = new byte[segmentsToLoad * _segmentSize];
byte segmentIndex = segment;
for (int i = 0; i < segmentsToLoad; i++)
{
if (segmentIndex == _pointerSegment)
{
ulong nextPageId = BitConverter.ToUInt64(currentPage, _pageStore.PageSize - 8);
currentPage = _pageStore.Retrieve(nextPageId, profiler);
segmentIndex = 0;
}
Array.Copy(currentPage, segmentIndex * _segmentSize, buffer, i * _segmentSize, _segmentSize);
segmentIndex++;
}
return(Encoding.UTF8.GetString(buffer, 4, resourceLength));
}
}
/// <summary>
/// Commits all changed and new pages to the page store
/// </summary>
/// <param name="commitId">The transaction identifier for the commit</param>
/// <param name="profiler"></param>
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
EnsureOutputStream();
using (profiler.Step("PageStore.Commit"))
{
try
{
foreach (var entry in _modifiedPages.OrderBy(e => e.Key))
{
// TODO: Ensure we are writing the correct commit
entry.Value.Item1.Write(_outputStream, commitId);
lock (_pageCacheLock)
{
PageCache.Instance.InsertOrUpdate(_filePath, entry.Value.Item1);
}
}
_modifiedPages.Clear();
}
catch (Exception)
{
_modifiedPages.Clear();
throw;
}
CurrentTransactionId = commitId;
_outputStream.Flush();
_outputStream.Close();
_outputStream = null;
}
}
public INode GetNode(ulong nodeId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.GetNode"))
{
INode ret;
if (_modifiedNodes.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return(ret);
}
if (_nodeCache.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return(ret);
}
profiler.Incr("NodeCache Miss");
using (profiler.Step("Load Node"))
{
var nodePage = _pageStore.Retrieve(nodeId, profiler);
var header = BitConverter.ToInt32(nodePage, 0);
if (header < 0)
{
ret = new InternalNode(nodeId, nodePage, ~header, _config);
}
else
{
ret = new LeafNode(nodeId, nodePage, header, _config);
}
_nodeCache.Add(ret);
return(ret);
}
}
}
private BinaryFilePage GetPage(ulong pageId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.GetPage"))
{
lock (_pageCacheLock)
{
BinaryFilePage page;
Tuple <BinaryFilePage, ulong> modifiedPage;
if (_modifiedPages.TryGetValue(pageId, out modifiedPage))
{
profiler.Incr("PageCache Hit");
return(modifiedPage.Item1);
}
page = PageCache.Instance.Lookup(_filePath, pageId) as BinaryFilePage;
if (page != null)
{
profiler.Incr("PageCache Hit");
return(page);
}
using (profiler.Step("Load Page"))
{
profiler.Incr("PageCache Miss");
page = new BinaryFilePage(_inputStream, pageId, _nominalPageSize);
PageCache.Instance.InsertOrUpdate(_filePath, page);
}
return(page);
}
}
}
public IEnumerable <KeyValuePair <byte[], byte []> > Scan(BrightstarProfiler profiler)
{
using (profiler.Step("Scan Entire BTree"))
{
return(Scan(_root, profiler));
}
}
public void TestSimpleJoinQuery()
{
var sparqlQuery =
@"PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/>
SELECT ?review WHERE {
?review bsbm:reviewFor ?product .
?product bsbm:productFeature <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature2330> .
} LIMIT 3";
// Warm-up
var client = BrightstarService.GetClient("type=embedded;storesDirectory=stores");
for (int i = 0; i < 5; i++)
{
client.ExecuteQuery("SparqlPerformance", sparqlQuery);
}
// Profile
var profiler = new BrightstarProfiler("SimpleJoinQuery");
for (int i = 0; i < 100; i++)
{
using (profiler.Step("ExecuteQuery"))
{
client.ExecuteQuery("SparqlPerformance", sparqlQuery);
}
}
Console.WriteLine(profiler.GetLogString());
}
/// <summary>
/// Performs the actual load of prefixes from a page.
/// </summary>
/// <param name="page"></param>
/// <param name="profiler"></param>
/// <remarks>Calls to this method should be made inside a critical section of code protected with a mutex or reader/writer lock</remarks>
private void InterlockedLoad(IPage page, BrightstarProfiler profiler)
{
using (profiler.Step("PrefixManager.InterlockedLoad"))
{
int offset = 0;
while (offset < _pageStore.PageSize)
{
ushort prefixLength = BitConverter.ToUInt16(page.Data, offset);
offset += 2;
if (prefixLength == ushort.MaxValue)
{
ulong nextPageId = BitConverter.ToUInt64(page.Data, offset);
if (nextPageId == 0)
{
// End of data
return;
}
page = _pageStore.Retrieve(nextPageId, profiler);
offset = 0;
}
else
{
var prefix = Encoding.UTF8.GetString(page.Data, offset, prefixLength);
offset += prefixLength;
var uriLen = BitConverter.ToUInt16(page.Data, offset);
offset += 2;
var uri = Encoding.UTF8.GetString(page.Data, offset, uriLen);
offset += uriLen;
_prefixMappings[uri] = prefix;
_shortValueMappings[prefix] = uri;
}
}
}
}
private IResource CreateLongUriResource(ulong txnId, string uri, BrightstarProfiler profiler)
{
ulong pageId;
byte segId;
_resourceTable.Insert(txnId, uri, out pageId, out segId, profiler);
return(new LongUriResource(uri, pageId, segId));
}
/// <summary>
/// Creates a new store writing triple sink
/// </summary>
/// <param name="writeStore">The store to add the triples to</param>
/// <param name="jobId">The unique identifier of the job that is writing to the store. May be Guid.Empty if the write is not part of any job.</param>
/// <param name="batchSize">Number of triples to insert per batch</param>
/// <param name="commitEachBatch">If true, then the inserts are committed to the store after each batch; if false then the server memory load is checked at the end of each batch and inserts are flushed only if the load exceeds a threshold (currently 80% of available physical RAM).</param>
/// <param name="profiler"></param>
public StoreTripleSink(IStore writeStore, Guid jobId, int batchSize = 10000, bool commitEachBatch = false, BrightstarProfiler profiler = null)
{
_batchSize = batchSize;
_store = writeStore;
_jobId = jobId;
_commitEachBatch = commitEachBatch;
_profiler = profiler;
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
var targetConfiguration = new BPlusTreeConfiguration(pageStore, Configuration.KeySize,
Configuration.ValueSize, Configuration.PageSize);
var builder = new BPlusTreeBuilder(pageStore, targetConfiguration);
return(builder.Build(transactionId, Scan(profiler), profiler));
}
/// <summary>
/// Creates a new store writing triple sink
/// </summary>
/// <param name="writeStore">The store to add the triples to</param>
/// <param name="jobId">The unique identifier of the job that is writing to the store. May be Guid.Empty if the write is not part of any job.</param>
/// <param name="batchSize">Number of triples to insert per batch</param>
/// <param name="commitEachBatch">If true, then the inserts are committed to the store after each batch; if false then the server memory load is checked at the end of each batch and inserts are flushed only if the load exceeds a threshold (currently 80% of available physical RAM).</param>
/// <param name="profiler"></param>
public StoreTripleSink(IStore writeStore, Guid jobId, int batchSize = 10000, bool commitEachBatch = false, BrightstarProfiler profiler = null)
{
_batchSize = batchSize;
_store = writeStore;
_jobId = jobId;
_commitEachBatch = commitEachBatch;
_profiler = profiler;
}
public void WarmupPageCache(int pagesToPreload, BrightstarProfiler profiler = null)
{
int totalLoaded = _subjectRelatedResourceIndex.Preload(pagesToPreload / 3, profiler);
totalLoaded += _objectRelatedResourceIndex.Preload(pagesToPreload - totalLoaded, profiler);
totalLoaded += _resourceIndex.Preload(pagesToPreload - totalLoaded, profiler);
_resourceTable.Preload(pagesToPreload - totalLoaded, profiler);
}
public virtual ulong Save(ulong transactionId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.Save"))
{
_isDirty = false;
return(RootId);
}
}
/// <summary>
/// Finds or creates a new ID for the graph with the specified graph URI
/// </summary>
/// <param name="graphUri">The graph URI to lookup</param>
/// <param name="profiler"></param>
/// <returns>The ID assigned to the graph</returns>
public int AssertGraphId(string graphUri, BrightstarProfiler profiler = null)
{
if (String.IsNullOrEmpty(graphUri))
{
throw new ArgumentException("Graph URI must not be null or an empty string", "graphUri");
}
if (graphUri.Length > short.MaxValue)
{
throw new ArgumentException(
String.Format("Graph URI string exceeds maximum allowed length of {0} bytes", short.MaxValue), "graphUri");
}
#if WINDOWS_PHONE || PORTABLE
lock (_lock)
{
int entryId;
if (_graphUriIndex.TryGetValue(graphUri, out entryId) && !_allEntries[entryId].IsDeleted)
{
return(entryId);
}
var newId = _allEntries.Count;
var entry = new GraphIndexEntry(newId, graphUri, false);
_allEntries.Add(entry);
_graphUriIndex.Add(graphUri, newId);
return(newId);
}
#else
using (profiler.Step("Assert Graph Id"))
{
_lock.EnterUpgradeableReadLock();
try
{
int entryId;
if (_graphUriIndex.TryGetValue(graphUri, out entryId) && !_allEntries[entryId].IsDeleted)
{
return(entryId);
}
_lock.EnterWriteLock();
try
{
var newId = _allEntries.Count;
var entry = new GraphIndexEntry(newId, graphUri, false);
_allEntries.Add(entry);
_graphUriIndex.Add(graphUri, newId);
return(newId);
}
finally
{
_lock.ExitWriteLock();
}
}
finally
{
_lock.ExitUpgradeableReadLock();
}
}
#endif
}
/// <summary>
/// Opens an existing tree in the page store
/// </summary>
/// <param name="pageStore"></param>
/// <param name="rootPageId">The page ID of the BTree root node</param>
/// <param name="keySize"></param>
/// <param name="dataSize"></param>
/// <param name="profiler"></param>
public BPlusTree(IPageStore pageStore, ulong rootPageId, int keySize = 8, int dataSize = 64, BrightstarProfiler profiler = null)
{
_config = new BPlusTreeConfiguration(keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
_modifiedNodes = new Dictionary<ulong, INode>();
_nodeCache = new WeakReferenceNodeCache();
_root = GetNode(rootPageId, profiler);
_nodeCache.Add(_root);
}
public IEnumerable <string> GetPredicates(BrightstarProfiler profiler = null)
{
return
(from resource in
_subjectRelatedResourceIndex.EnumeratePredicates(profiler)
.Select(rid => _resourceIndex.GetResource(rid))
where resource != null && !resource.IsLiteral
select _prefixManager.ResolvePrefixedUri(resource.Value));
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
using (profiler.Step("RelatedResourceIndex.Write"))
{
var indexBuilder = new BPlusTreeBuilder(pageStore, Configuration);
return(indexBuilder.Build(transactionId, WritePredicateIndexes(pageStore, transactionId, profiler),
profiler));
}
}
public ulong Build(ulong txnId, IEnumerable<KeyValuePair<byte[], byte []>> orderedValues, BrightstarProfiler profiler = null)
{
var nodeList = MakeInternalNodes(txnId, MakeLeafNodes(txnId, orderedValues.GetEnumerator(), profiler), profiler).ToList();
while(nodeList.Count > 1)
{
nodeList = MakeInternalNodes(txnId, nodeList, profiler).ToList();
}
return nodeList[0].Value;
}
public int Preload(int numPages, BrightstarProfiler profiler)
{
var maxPages = Math.Min(_stream.Length / PageSize, numPages);
for (int pageId = 0; pageId < maxPages; pageId++)
{
Retrieve((ulong)pageId, profiler);
}
return((int)maxPages);
}
public ConcurrentGraphIndex(IPageStore pageStore, ulong rootPage, BrightstarProfiler profiler)
{
using (profiler.Step("Load ConcurrentGraphIndex"))
{
_pageStore = pageStore;
_graphUriIndex = new Dictionary <string, int>();
_allEntries = new List <GraphIndexEntry>();
Read(rootPage, profiler);
}
}
public ulong GetTripleCount(string predicateUri, BrightstarProfiler profiler = null)
{
var predicateId = _resourceIndex.GetResourceId(_prefixManager.MakePrefixedUri(predicateUri));
if (predicateId == StoreConstants.NullUlong)
{
return(0L);
}
return(_subjectRelatedResourceIndex.CountPredicateRelationships(predicateId, profiler));
}
public ConcurrentGraphIndex(IPageStore pageStore, ulong rootPage, BrightstarProfiler profiler)
{
using (profiler.Step("Load ConcurrentGraphIndex"))
{
_pageStore = pageStore;
_graphUriIndex = new Dictionary<string, int>();
_allEntries = new List<GraphIndexEntry>();
Read(rootPage, profiler);
}
}
public static IPageStore OpenPageStore(string fileName, bool readOnly, PersistenceType persistenceType = PersistenceType.AppendOnly, ulong txnId = 1UL, BrightstarProfiler profiler = null)
{
using (profiler.Step("Open Page Store"))
{
if (persistenceType == PersistenceType.AppendOnly)
{
return new AppendOnlyFilePageStore(PersistenceManager, fileName, 4096, readOnly, false);
}
return new BinaryFilePageStore(PersistenceManager, fileName, 4096, readOnly, txnId);
}
}
private void StartNewPage(ulong transactionId, BrightstarProfiler profiler)
{
ulong nextPage = _pageStore.Create();
if (_currentPage > 0)
{
_pageStore.Write(transactionId, _currentPage, BitConverter.GetBytes(nextPage), 0, _pageStore.PageSize - 8, 8, profiler);
}
_currentPage = nextPage;
_nextSegment = 0;
}
public Store(string storeLocation, IPageStore dataPageStore, IResourceTable resourceTable, ulong storePageId, BrightstarProfiler profiler)
{
using (profiler.Step("Load Store"))
{
DirectoryPath = storeLocation;
_pageStore = dataPageStore;
_resourceTable = resourceTable;
var storePage = _pageStore.Retrieve(storePageId, profiler);
Load(storePage, profiler);
}
}
/// <summary>
/// Retrieves the data for the specified page
/// </summary>
/// <param name="pageId">The ID of the page</param>
/// <param name="profiler"></param>
/// <returns>The data buffer for the page</returns>
public IPage Retrieve(ulong pageId, BrightstarProfiler profiler)
{
Tuple <BinaryFilePage, ulong> modifiedPage;
if (_modifiedPages.TryGetValue(pageId, out modifiedPage))
{
return(new BinaryPageAdapter(this, modifiedPage.Item1, modifiedPage.Item2, true));
}
var page = GetPage(pageId, profiler);
return(page == null ? null : new BinaryPageAdapter(this, page, CurrentTransactionId, false));
}
public IResource CreateNew(ulong txnId, string resourceValue, bool isLiteral, ulong dataTypeId, ulong langCodeId, BrightstarProfiler profiler)
{
var valueLength = Encoding.UTF8.GetByteCount(resourceValue);
if (isLiteral)
{
return valueLength <= MaxLocalLiteralLength
? new ShortLiteralResource(resourceValue, dataTypeId, langCodeId)
: CreateLongLiteralResource(txnId, resourceValue, dataTypeId, langCodeId, profiler);
}
return valueLength < MaxLocalUriLength
? new ShortUriResource(resourceValue)
: CreateLongUriResource(txnId, resourceValue, profiler);
}
public override void Run()
{
try
{
Logging.LogInfo("Import job being run on file " + _contentFileName);
StoreWorker.TransactionLog.LogStartTransaction(this);
var parser = GetParser(_contentFileName);
var storeDirectory = StoreWorker.WriteStore.DirectoryPath;
var filePath = Path.Combine(storeDirectory,
".." + Path.DirectorySeparatorChar + "import" + Path.DirectorySeparatorChar +
_contentFileName);
var profiler = new BrightstarProfiler("Import " + _contentFileName); // TODO : Conditionally create this if profiling is enabled
Logging.LogDebug("Import file path calculated as '{0}'", filePath);
if (!File.Exists(filePath))
{
ErrorMessage = String.Format("Cannot find file {0} in import directory", _contentFileName);
throw new FileNotFoundException(ErrorMessage);
}
using (_fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
_importTripleSink = new StoreTripleSink(StoreWorker.WriteStore, JobId,
Configuration.TransactionFlushTripleCount,
profiler:profiler);
parser.Parse(_fileStream, this, _graphUri);
}
StoreWorker.WriteStore.Commit(JobId, profiler);
StoreWorker.InvalidateReadStore();
Logging.LogInfo("Import job completed successfully for " + _contentFileName);
if (profiler != null)
{
Logging.LogInfo(profiler.GetLogString());
}
StoreWorker.TransactionLog.LogEndSuccessfulTransaction(this);
}
catch (RdfParserException parserException)
{
ErrorMessage = parserException.Message;
ExceptionDetail = new ExceptionDetail(parserException);
Logging.LogInfo("Parser error processing import job on file " + _contentFileName + ". " + parserException.Message);
throw;
}
catch (Exception ex)
{
ErrorMessage = "Error importing file " + _contentFileName + ". " + ex.Message;
StoreWorker.TransactionLog.LogEndFailedTransaction(this);
Logging.LogInfo("Error processing import job on file " + _contentFileName + ". Error Message: " + ex.Message + " Stack trace: " + ex.StackTrace);
throw;
}
}
public static IPageStore CreateEmptyPageStore(string fileName, PersistenceType persistenceType = PersistenceType.AppendOnly, BrightstarProfiler profiler = null)
{
using (profiler.Step("Create Empty Page Store"))
{
using (profiler.Step("Delete Existing"))
{
if (PersistenceManager.FileExists(fileName)) PersistenceManager.DeleteFile(fileName);
//if (File.Exists(fileName)) File.Delete(fileName);
}
using (profiler.Step("Create New Page Store"))
{
if (persistenceType == PersistenceType.AppendOnly)
{
return new AppendOnlyFilePageStore(PersistenceManager, fileName, 4096, false, false);
}
return new BinaryFilePageStore(PersistenceManager, fileName, 4096, false, 1);
}
}
}
/// <summary>
/// Adds a related resource index entry
/// </summary>
/// <param name="txnId"> </param>
/// <param name="resourceId">The resource ID of the "start" resource</param>
/// <param name="predicateId">The resource ID of the predicate that relates the resources</param>
/// <param name="relatedResourceId">The resource ID of the "related" resource</param>
/// <param name="graphId">The resource ID of the graph containing the relationship</param>
/// <param name="profiler"></param>
public void AddRelatedResource(ulong txnId, ulong resourceId, ulong predicateId, ulong relatedResourceId, int graphId, BrightstarProfiler profiler = null)
{
using (profiler.Step("Add Related Resource"))
{
var predicateIndex = AssertPredicateIndex(txnId, predicateId, profiler);
var key = MakePredicateIndexKey(resourceId, graphId, relatedResourceId);
try
{
using (profiler.Step("Insert Into Predicate Index"))
{
predicateIndex.Insert(txnId, key, null, profiler: profiler);
}
}
catch (DuplicateKeyException)
{
// Ignore duplicate key exceptions
}
}
}
/// <summary>
/// Enumerates the resources related to a start resource by a specific predicate
/// </summary>
/// <param name="resourceId">The resource ID of the start resource</param>
/// <param name="predicateId">The resource ID of the predicate that relates the resources. If set to Constants.NullUlong, returns relationships for all predicates</param>
/// <param name="graphId">The resource ID of the graph containing the relationships. If set to Constants.NullUlong, returns relationships from all graphs</param>
/// <param name="profiler"></param>
/// <returns></returns>
public IEnumerable<IRelatedResource> EnumerateRelatedResources(ulong resourceId, ulong predicateId = StoreConstants.NullUlong, int graphId = -1, BrightstarProfiler profiler = null)
{
using (profiler.Step("EnumerateRelatedResources"))
{
if (predicateId != StoreConstants.NullUlong)
{
byte[] minKey = MakePredicateIndexKey(resourceId,
graphId < 0 ? 0 : graphId,
UInt64.MinValue);
byte[] maxKey = MakePredicateIndexKey(resourceId,
graphId < 0 ? Int32.MaxValue : graphId,
UInt64.MaxValue);
var predicateIndex = GetPredicateIndex(predicateId, profiler);
if (predicateIndex != null)
{
foreach (var r in predicateIndex.Scan(minKey, maxKey, profiler).Select(
x =>
new RelatedResource(predicateId, GetGraphIdFromKey(x.Key),
GetRelatedResourceIdFromKey(x.Key))))
{
yield return r;
}
}
}
else
{
foreach (var entry in Scan(0ul, UInt64.MaxValue, profiler))
{
// Load the predicate index into the cache
GetPredicateIndex(entry.Key,BitConverter.ToUInt64(entry.Value, 0), profiler);
// Then a recursive call to enumerate the index tree
foreach (var r in EnumerateRelatedResources(resourceId, entry.Key, graphId, profiler))
{
yield return r;
}
}
}
}
}
/// <summary>
/// Ensures that the specified resource is in the resource index and returns its resource ID
/// </summary>
/// <param name="txnId">The ID of the current update transaction</param>
/// <param name="resourceValue">The resource string value</param>
/// <param name="isLiteral">Boolean flag indicating if the resource is a literal (true) or a URI (false)</param>
/// <param name="dataType">The resource data-type URI</param>
/// <param name="langCode">The resource language string</param>
/// <param name="addToCache">Boolean flag indicating if newly indexed resources should be added to the local cache</param>
/// <param name="profiler"></param>
/// <returns>The resource ID for the resource</returns>
public ulong AssertResourceInIndex(ulong txnId, string resourceValue, bool isLiteral = false, string dataType = null, string langCode = null, bool addToCache = true, BrightstarProfiler profiler = null)
{
using (profiler.Step("AssertResourceInIndex"))
{
// Normalize language code to null if it is an empty string
if (String.IsNullOrEmpty(langCode))
{
langCode = null;
}
// Retrieve the resource ID for the datatype URI (if any)
var dataTypeId = String.IsNullOrEmpty(dataType)
? StoreConstants.NullUlong
: AssertResourceInIndex(txnId, dataType, profiler:profiler);
var hashString = isLiteral ? MakeHashString(resourceValue, dataType, langCode) : resourceValue;
ulong resourceId;
if (_resourceIdCache.TryGetValue(hashString, out resourceId))
{
return resourceId;
}
// Get a ulong resource ID for the language code string
var langCodeId = String.IsNullOrEmpty(langCode)
? StoreConstants.NullUlong
: AssertResourceInIndex(txnId, langCode, true, profiler:profiler);
resourceId = AssertResourceInBTree(txnId, resourceValue, isLiteral, dataTypeId, langCodeId,
StringExtensions.GetBrightstarHashCode(hashString), profiler);
if (addToCache)
{
_resourceIdCache.Add(hashString, resourceId);
}
return resourceId;
}
}
/// <summary>
/// Insert a triple into the store
/// </summary>
/// <param name="subject"></param>
/// <param name="predicate"></param>
/// <param name="objValue"></param>
/// <param name="isObjectLiteral"></param>
/// <param name="dataType"></param>
/// <param name="langCode"></param>
/// <param name="graphUri"></param>
/// <param name="profiler"></param>
public void InsertTriple(string subject, string predicate, string objValue, bool isObjectLiteral, string dataType, string langCode, string graphUri, BrightstarProfiler profiler = null)
{
using (profiler.Step("InsertTriple"))
{
using (profiler.Step("Normalization"))
{
// Normalize subject, predicate, objValue (if it is not a literal), dataType (if it is not null)
// and graphUri (if it is not null)
try
{
subject = new Uri(subject, UriKind.Absolute).ToString();
}
catch (FormatException)
{
throw new InvalidTripleException(
String.Format("The subject '{0}' could not be parsed as a valid URI.", subject));
}
try
{
predicate = new Uri(predicate, UriKind.Absolute).ToString();
}
catch (FormatException)
{
throw new InvalidTripleException(
String.Format("The predicate'{0}' could not be parsed as a valid URI.", predicate));
}
if (!isObjectLiteral)
{
try
{
objValue = new Uri(objValue, UriKind.Absolute).ToString();
}
catch (FormatException)
{
throw new InvalidTripleException(
String.Format("The object '{0}' could not be parsed as a valid URI.", objValue));
}
}
if (isObjectLiteral && !String.IsNullOrEmpty(dataType))
{
try
{
dataType = new Uri(dataType, UriKind.Absolute).ToString();
}
catch (FormatException)
{
throw new InvalidTripleException(
String.Format("The dataType '{0}' could not be parsed as a valid URI.", dataType));
}
}
if (!String.IsNullOrEmpty(graphUri))
{
try
{
graphUri = new Uri(graphUri, UriKind.Absolute).ToString();
}
catch (FormatException)
{
throw new InvalidTripleException(
String.Format("The graphUri '{0}' could not be parsed as a valid URI.", graphUri));
}
}
if (isObjectLiteral && dataType == null)
{
dataType = RdfDatatypes.PlainLiteral;
}
// Normalize language code to lower-case (per http://www.w3.org/TR/rdf-concepts/#section-Graph-Literal)
if (langCode != null)
{
langCode = langCode.ToLowerInvariant();
}
}
using (profiler.Step("Make Prefixed Uris"))
{
subject = _prefixManager.MakePrefixedUri(subject);
predicate = _prefixManager.MakePrefixedUri(predicate);
if (!isObjectLiteral)
{
objValue = _prefixManager.MakePrefixedUri(objValue);
}
}
var txnId = _currentTxnId + 1;
ulong sid = _resourceIndex.AssertResourceInIndex(txnId, subject, profiler:profiler);
ulong pid = _resourceIndex.AssertResourceInIndex(txnId, predicate, profiler:profiler);
ulong oid = _resourceIndex.AssertResourceInIndex(txnId, objValue, isObjectLiteral, dataType, langCode,
!isObjectLiteral, profiler);
int gid = _graphIndex.AssertGraphId(graphUri, profiler);
_subjectRelatedResourceIndex.AddRelatedResource(txnId, sid, pid, oid, gid, profiler);
_objectRelatedResourceIndex.AddRelatedResource(txnId, oid, pid, sid, gid, profiler);
}
}
void Read(ulong rootPageId, BrightstarProfiler profiler)
{
byte[] currentPage = _pageStore.Retrieve(rootPageId, profiler);
int offset = 0;
int entryIndex = 0;
while(true)
{
var marker = currentPage[offset++];
if (marker == 0xff)
{
ulong nextPageId = BitConverter.ToUInt64(currentPage, _pageStore.PageSize - 8);
if (nextPageId == 0) return;
currentPage = _pageStore.Retrieve(nextPageId, profiler);
offset = 0;
}
else if (marker == 2)
{
_allEntries.Add(new GraphIndexEntry(entryIndex++, null, true));
}
else
{
int uriByteLength = BitConverter.ToInt32(currentPage, offset);
offset += 4;
var uri = Encoding.UTF8.GetString(currentPage, offset, uriByteLength);
offset += uriByteLength;
var newEntry = new GraphIndexEntry(entryIndex++, uri, marker == 1);
_allEntries.Add(newEntry);
if (!newEntry.IsDeleted) _graphUriIndex[newEntry.Uri] = newEntry.Id;
}
}
}
private ulong Save(BrightstarProfiler profiler)
{
using (profiler.Step("Store.Save"))
{
_resourceTable.Commit(_currentTxnId + 1, profiler);
var txnId = _currentTxnId + 1;
var graphIndexId = _graphIndex.Save(txnId, profiler);
var prefixManagerId = _prefixManager.Save(txnId, profiler);
var resourceIndexId = _resourceIndex.Save(txnId, profiler);
var subjectRelatedResourceIndexId = _subjectRelatedResourceIndex.Save(txnId, profiler);
var objectRelatedResourceIndexId = _objectRelatedResourceIndex.Save(txnId, profiler);
var buff = CreateStoreHeader(graphIndexId, prefixManagerId, resourceIndexId,
subjectRelatedResourceIndexId, objectRelatedResourceIndexId);
var page = _pageStore.Create(txnId);
page.SetData(buff);
page.SetData(buff, 0, 128);
_pageStore.Commit(txnId, profiler);
return page.Id;
}
}
/*
* Data format for the store page:
*
* 00-03: Store format version (int)
* 04-11: Transaction id (ulong)
* 12-19: Graph Index start page id (ulong)
* 20-27: Prefix Manager start page id (ulong)
* 28-35: Resource Index start page id (ulong)
* 36-43: Subject Related Resource Index start page id (ulong)
* 44-51: Object Related Resource Index start page id (ulong)
* 44-107: Reserved (all zeros for now)
* 108-127: SHA1 hash of bytes 00-108
* 128-255: Repeat of the above structure
*/
private bool Load(IPage storePage, BrightstarProfiler profiler)
{
using (profiler.Step("Store.Load"))
{
// Validate the hash for the index bloc
using (var sha1 = new SHA1Managed())
{
var recordedHash = new byte[20];
Array.Copy(storePage.Data, 108, recordedHash, 0, 20);
var calculatedHash = sha1.ComputeHash(storePage.Data, 0, 108);
if (recordedHash.Compare(calculatedHash) != 0)
{
return false;
}
}
// Load indexes from the pointers
int storeVersion = BitConverter.ToInt32(storePage.Data, 0);
if (storeVersion == 1)
{
_currentTxnId = BitConverter.ToUInt64(storePage.Data, 4);
var graphIndexId = BitConverter.ToUInt64(storePage.Data, 12);
_graphIndex = new ConcurrentGraphIndex(_pageStore, graphIndexId, profiler);
var prefixManagerId = BitConverter.ToUInt64(storePage.Data, 20);
_prefixManager = new PrefixManager(_pageStore, prefixManagerId, profiler);
var resourceIndexId = BitConverter.ToUInt64(storePage.Data, 28);
_resourceIndex = new ResourceIndex.ResourceIndex(_pageStore, _resourceTable, resourceIndexId);
var relatedResourceIndexId = BitConverter.ToUInt64(storePage.Data, 36);
_subjectRelatedResourceIndex = new RelatedResourceIndex.RelatedResourceIndex(_pageStore,
relatedResourceIndexId, profiler);
var objectRelatedResourceIndexId = BitConverter.ToUInt64(storePage.Data, 44);
_objectRelatedResourceIndex = new RelatedResourceIndex.RelatedResourceIndex(_pageStore,
objectRelatedResourceIndexId, profiler);
}
return true;
}
}
public ulong GetTripleCount(string predicateUri, BrightstarProfiler profiler = null)
{
var predicateId = _resourceIndex.GetResourceId(_prefixManager.MakePrefixedUri(predicateUri));
if (predicateId == StoreConstants.NullUlong) return 0L;
return _subjectRelatedResourceIndex.CountPredicateRelationships(predicateId, profiler);
}
public IEnumerable<string> GetPredicates(BrightstarProfiler profiler = null)
{
return
from resource in
_subjectRelatedResourceIndex.EnumeratePredicates(profiler)
.Select(rid => _resourceIndex.GetResource(rid))
where resource != null && !resource.IsLiteral
select _prefixManager.ResolvePrefixedUri(resource.Value);
}
public void Commit(Guid jobId, BrightstarProfiler profiler = null)
{
using (profiler.Step("Store Commit"))
{
ulong storePageId = Save(profiler);
var storeManager = StoreManagerFactory.GetStoreManager();
var mf = storeManager.GetMasterFile(DirectoryPath);
mf.AppendCommitPoint(new CommitPoint(storePageId,_currentTxnId + 1,DateTime.UtcNow, jobId));
_currentTxnId++;
}
}
/// <summary>
/// Commits all updates so far to the store and returns the start position of the Store object in the store file.
/// </summary>
public void FlushChanges(BrightstarProfiler profiler = null)
{
_subjectRelatedResourceIndex.FlushCache();
_objectRelatedResourceIndex.FlushCache();
}
