public IPage Retrieve(ulong pageId, BrightstarProfiler profiler)
{
using (profiler.Step("BinaryFilePageStore.GetPage"))
{
// Look in the page cache
IPage page = PageCache.Instance.Lookup(_partitionId, pageId) as BinaryFilePage;
if (page != null)
{
profiler.Incr("PageCache Hit");
return(page);
}
// See if the page is queued for writing
if (_backgroundPageWriter != null && _backgroundPageWriter.TryGetPage(pageId, out page))
{
profiler.Incr("BackgroundWriter Queue Hit");
return(page);
}
// Not found in memory, so go to the disk
profiler.Incr("PageCache Miss");
using (profiler.Step("Load Page"))
{
page = _modifiedPages.ContainsKey(pageId)
? new BinaryFilePage(_inputStream, pageId, _nominalPageSize, _writeTxnId, true)
: new BinaryFilePage(_inputStream, pageId, _nominalPageSize, _readTxnId, false);
PageCache.Instance.InsertOrUpdate(_partitionId, page);
return(page);
}
}
}
public byte[] Retrieve(ulong pageId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Retrieve"))
{
if (!_readonly && pageId >= _newPageOffset)
{
var newPage = _newPages[(int) (pageId - _newPageOffset)];
return newPage.Data;
}
var page = PageCache.Instance.Lookup(_path, pageId) as FilePage;
if (page != null)
{
profiler.Incr("PageCache Hit");
return page.Data;
}
using (profiler.Step("Load Page"))
{
profiler.Incr("PageCache Miss");
using (profiler.Step("Create FilePage"))
{
// Lock on stream to prevent attempts to concurrently load a page
lock (_stream)
{
page = new FilePage(_stream, pageId, _pageSize);
}
}
using (profiler.Step("Add FilePage To Cache"))
{
PageCache.Instance.InsertOrUpdate(_path, page);
}
return page.Data;
}
}
}
private BinaryFilePage GetPage(ulong pageId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.GetPage"))
{
lock (_pageCacheLock)
{
BinaryFilePage page;
Tuple <BinaryFilePage, ulong> modifiedPage;
if (_modifiedPages.TryGetValue(pageId, out modifiedPage))
{
profiler.Incr("PageCache Hit");
return(modifiedPage.Item1);
}
page = PageCache.Instance.Lookup(_filePath, pageId) as BinaryFilePage;
if (page != null)
{
profiler.Incr("PageCache Hit");
return(page);
}
using (profiler.Step("Load Page"))
{
profiler.Incr("PageCache Miss");
page = new BinaryFilePage(_inputStream, pageId, _nominalPageSize);
PageCache.Instance.InsertOrUpdate(_filePath, page);
}
return(page);
}
}
}
public INode GetNode(ulong nodeId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.GetNode"))
{
INode ret;
if (_nodeCache.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return ret;
}
profiler.Incr("NodeCache Miss");
using (profiler.Step("Load Node"))
{
var nodePage = _pageStore.Retrieve(nodeId, profiler);
var header = BitConverter.ToInt32(nodePage.Data, 0);
if (header < 0)
{
ret = MakeInternalNode(nodePage, ~header);
#if DEBUG_BTREE
_config.BTreeDebug("{0}: Loaded INTERNAL node from page {1}. {2}",_config.DebugId, nodePage.Id, ret.ToString());
#endif
}
else
{
ret = MakeLeafNode(nodePage, header);
#if DEBUG_BTREE
_config.BTreeDebug("{0}: Loaded LEAF node from page {1}. {2}", _config.DebugId, nodePage.Id, ret.ToString());
#endif
}
_nodeCache.Add(ret);
return ret;
}
}
}
public byte[] Retrieve(ulong pageId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Retrieve"))
{
if (!_readonly && pageId >= _newPageOffset)
{
var newPage = _newPages[(int)(pageId - _newPageOffset)];
return(newPage.Data);
}
var page = PageCache.Instance.Lookup(_path, pageId) as FilePage;
if (page != null)
{
profiler.Incr("PageCache Hit");
return(page.Data);
}
using (profiler.Step("Load Page"))
{
profiler.Incr("PageCache Miss");
using (profiler.Step("Create FilePage"))
{
// Lock on stream to prevent attempts to concurrently load a page
lock (_stream)
{
page = new FilePage(_stream, pageId, _pageSize);
}
}
using (profiler.Step("Add FilePage To Cache"))
{
PageCache.Instance.InsertOrUpdate(_path, page);
}
return(page.Data);
}
}
}
//private static ulong _targetResourceId = 15806097072303636481;
private ulong AssertResourceInBTree(ulong txnId, string resourceValue, bool isLiteral, ulong dataTypeId, ulong langCodeId, uint hashCode, BrightstarProfiler profiler)
{
using (profiler.Step("AssertResourceInBTree"))
{
ulong rangeMin = MakeId(hashCode, 0), rangeMax = MakeId(hashCode, UInt32.MaxValue);
ulong lastResourceId = StoreConstants.NullUlong;
foreach (var indexEntry in this.Scan(rangeMin, rangeMax, profiler))
{
var resource = _resourceStore.FromBTreeValue(indexEntry.Value);
if (resource.Matches(resourceValue, isLiteral, dataTypeId, langCodeId))
{
return(indexEntry.Key);
}
lastResourceId = indexEntry.Key;
}
// If we reach here, there has been no match so we need to insert a new resource
using (profiler.Step("Add Resource To BTree"))
{
var newResource = _resourceStore.CreateNew(txnId, resourceValue, isLiteral, dataTypeId, langCodeId, profiler);
var resourceKey = MakeId(hashCode, GetBucketPosition(lastResourceId) + 1);
Insert(txnId, resourceKey, newResource.GetData());
return(resourceKey);
}
}
}
public INode GetNode(ulong nodeId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.GetNode"))
{
INode ret;
if (_modifiedNodes.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return(ret);
}
if (_nodeCache.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return(ret);
}
profiler.Incr("NodeCache Miss");
using (profiler.Step("Load Node"))
{
var nodePage = _pageStore.Retrieve(nodeId, profiler);
var header = BitConverter.ToInt32(nodePage, 0);
if (header < 0)
{
ret = new InternalNode(nodeId, nodePage, ~header, _config);
}
else
{
ret = new LeafNode(nodeId, nodePage, header, _config);
}
_nodeCache.Add(ret);
return(ret);
}
}
}
public INode GetNode(ulong nodeId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.GetNode"))
{
INode ret;
if (_nodeCache.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return(ret);
}
profiler.Incr("NodeCache Miss");
using (profiler.Step("Load Node"))
{
var nodePage = _pageStore.Retrieve(nodeId, profiler);
var header = BitConverter.ToInt32(nodePage.Data, 0);
if (header < 0)
{
ret = MakeInternalNode(nodePage, ~header);
#if DEBUG_BTREE
_config.BTreeDebug("{0}: Loaded INTERNAL node from page {1}. {2}", _config.DebugId, nodePage.Id, ret.ToString());
#endif
}
else
{
ret = MakeLeafNode(nodePage, header);
#if DEBUG_BTREE
_config.BTreeDebug("{0}: Loaded LEAF node from page {1}. {2}", _config.DebugId, nodePage.Id, ret.ToString());
#endif
}
_nodeCache.Add(ret);
return(ret);
}
}
}
public IPage Retrieve(ulong pageId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Retrieve"))
{
if (!_readonly && pageId >= _newPageOffset)
{
var newPageRef = _newPages[(int)(pageId - _newPageOffset)];
if (newPageRef.IsAlive)
{
var newPage = newPageRef.Target as IPage;
if (newPage != null)
{
return(newPage);
}
}
}
var page = PageCache.Instance.Lookup(_path, pageId) as IPage;
if (page != null)
{
profiler.Incr("PageCache Hit");
return(page);
}
if (_backgroundPageWriter != null)
{
// See if the page is currently queued for background write
if (_backgroundPageWriter.TryGetPage(pageId, out page))
{
profiler.Incr("BackgroundWriter Queue Hit");
return(page);
}
}
using (profiler.Step("Load Page"))
{
profiler.Incr("PageCache Miss");
using (profiler.Step("Create FilePage"))
{
page = new FilePage(_stream, pageId, _pageSize);
if (_backgroundPageWriter != null)
{
_backgroundPageWriter.ResetTimestamp(pageId);
}
#if DEBUG_PAGESTORE
Logging.LogDebug("Load {0} {1}", pageId, BitConverter.ToInt32(page.Data, 0));
#endif
}
using (profiler.Step("Add FilePage To Cache"))
{
PageCache.Instance.InsertOrUpdate(_path, page);
}
return(page);
}
}
}
public void Insert(ulong txnId, byte[] key, byte[] value, bool overwrite = false, BrightstarProfiler profiler = null)
{
using (profiler.Step("LeafNode.Insert"))
{
if (IsFull)
{
throw new NodeFullException();
}
int insertIndex = Search(key);
if (insertIndex >= 0)
{
if (overwrite)
{
EnsureWriteable(txnId);
_page.SetData(value, 0,
ValueOffset(insertIndex),
Math.Min(value.Length, _config.ValueSize));
return;
}
throw new DuplicateKeyException();
}
EnsureWriteable(txnId);
insertIndex = ~insertIndex;
RightShiftFrom(insertIndex, 1);
_page.SetData(key, 0, KeyOffset(insertIndex), _config.KeySize);
if (_config.ValueSize > 0 && value != null)
{
_page.SetData(value, 0, ValueOffset(insertIndex), Math.Min(_config.ValueSize, value.Length));
}
KeyCount++;
#if DEBUG_BTREE
_config.BTreeDebug("LeafNode.Insert@{0}. Key={1}. Updated Node: {2}", PageId, key.Dump(), DumpKeys());
#endif
}
}
public void TestSimpleJoinQuery()
{
var sparqlQuery =
@"PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/>
SELECT ?review WHERE {
?review bsbm:reviewFor ?product .
?product bsbm:productFeature <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature2330> .
} LIMIT 3";
// Warm-up
var client = BrightstarService.GetClient("type=embedded;storesDirectory=stores");
for (int i = 0; i < 5; i++)
{
client.ExecuteQuery("SparqlPerformance", sparqlQuery);
}
// Profile
var profiler = new BrightstarProfiler("SimpleJoinQuery");
for (int i = 0; i < 1000; i++)
{
using (profiler.Step("ExecuteQuery"))
{
client.ExecuteQuery("SparqlPerformance", sparqlQuery);
}
}
Console.WriteLine(profiler.GetLogString());
}
/// <summary>
/// Retrieve the resource at the specified page and segment offset
/// </summary>
/// <param name="pageId">The ID of the page that holds the resource to be retrieved</param>
/// <param name="segment">The index of the segment within the page that holds the start of the resource</param>
/// <param name="profiler"></param>
/// <returns>The resource</returns>
public string GetResource(ulong pageId, byte segment, BrightstarProfiler profiler)
{
using (profiler.Step("ResourceTable.GetResource"))
{
var currentPage = _pageStore.Retrieve(pageId, profiler);
int resourceLength = BitConverter.ToInt32(currentPage, segment*_segmentSize);
int totalLength = resourceLength + 4;
int segmentsToLoad = totalLength/_segmentSize;
if (totalLength%_segmentSize > 0) segmentsToLoad++;
var buffer = new byte[segmentsToLoad*_segmentSize];
byte segmentIndex = segment;
for (int i = 0; i < segmentsToLoad; i++)
{
if (segmentIndex == _pointerSegment)
{
ulong nextPageId = BitConverter.ToUInt64(currentPage, _pageStore.PageSize - 8);
currentPage = _pageStore.Retrieve(nextPageId, profiler);
segmentIndex = 0;
}
Array.Copy(currentPage, segmentIndex*_segmentSize, buffer, i*_segmentSize, _segmentSize);
segmentIndex++;
}
return Encoding.UTF8.GetString(buffer, 4, resourceLength);
}
}
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Commit"))
{
var pagesToWrite = _newPages.Where(rf => rf.IsAlive).Select(rf => rf.Target as IPage).Where(pg => pg != null);
if (_backgroundPageWriter != null)
{
foreach (var page in pagesToWrite)
{
_backgroundPageWriter.QueueWrite(page, commitId);
}
_backgroundPageWriter.Flush();
RestartBackgroundWriter();
}
else
{
using (var outputStream = _peristenceManager.GetOutputStream(_path, FileMode.Open))
{
foreach (var page in pagesToWrite)
{
page.Write(outputStream, commitId);
}
}
}
_newPages.Clear();
_newPageOffset = _nextPageId;
PageCache.Instance.Clear(_path);
}
}
public INode GetNode(ulong nodeId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.GetNode"))
{
INode ret;
if (_modifiedNodes.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return ret;
}
if (_nodeCache.TryGetValue(nodeId, out ret))
{
profiler.Incr("NodeCache Hit");
return ret;
}
profiler.Incr("NodeCache Miss");
using (profiler.Step("Load Node"))
{
var nodePage = _pageStore.Retrieve(nodeId, profiler);
var header = BitConverter.ToInt32(nodePage, 0);
if (header < 0)
{
ret = new InternalNode(nodeId, nodePage, ~header, _config);
}
else
{
ret = new LeafNode(nodeId, nodePage, header, _config);
}
_nodeCache.Add(ret);
return ret;
}
}
}
/// <summary>
/// Commits all changed and new pages to the page store
/// </summary>
/// <param name="commitId">The transaction identifier for the commit</param>
/// <param name="profiler"></param>
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
EnsureOutputStream();
using (profiler.Step("PageStore.Commit"))
{
try
{
foreach (var entry in _modifiedPages.OrderBy(e => e.Key))
{
// TODO: Ensure we are writing the correct commit
entry.Value.Item1.Write(_outputStream, commitId);
lock (_pageCacheLock)
{
PageCache.Instance.InsertOrUpdate(_filePath, entry.Value.Item1);
}
}
_modifiedPages.Clear();
}
catch (Exception)
{
_modifiedPages.Clear();
throw;
}
CurrentTransactionId = commitId;
_outputStream.Flush();
_outputStream.Close();
_outputStream = null;
}
}
/// <summary>
/// Performs the actual load of prefixes from a page.
/// </summary>
/// <param name="page"></param>
/// <param name="profiler"></param>
/// <remarks>Calls to this method should be made inside a critical section of code protected with a mutex or reader/writer lock</remarks>
private void InterlockedLoad(IPage page, BrightstarProfiler profiler)
{
using (profiler.Step("PrefixManager.InterlockedLoad"))
{
int offset = 0;
while (offset < _pageStore.PageSize)
{
ushort prefixLength = BitConverter.ToUInt16(page.Data, offset);
offset += 2;
if (prefixLength == ushort.MaxValue)
{
ulong nextPageId = BitConverter.ToUInt64(page.Data, offset);
if (nextPageId == 0)
{
// End of data
return;
}
page = _pageStore.Retrieve(nextPageId, profiler);
offset = 0;
}
else
{
var prefix = Encoding.UTF8.GetString(page.Data, offset, prefixLength);
offset += prefixLength;
var uriLen = BitConverter.ToUInt16(page.Data, offset);
offset += 2;
var uri = Encoding.UTF8.GetString(page.Data, offset, uriLen);
offset += uriLen;
_prefixMappings[uri] = prefix;
_shortValueMappings[prefix] = uri;
}
}
}
}
public void Insert(ulong transactionId, string resource, out ulong pageId, out byte segmentId, BrightstarProfiler profiler)
{
using (profiler.Step("ResourceTable.Insert"))
{
var byteCount = Encoding.UTF8.GetByteCount(resource);
var resourceBytes = new byte[byteCount + 4];
BitConverter.GetBytes(byteCount).CopyTo(resourceBytes, 0);
Encoding.UTF8.GetBytes(resource, 0, resource.Length, resourceBytes, 4);
lock (_writeLock)
{
if (_nextSegment == _pointerSegment)
{
StartNewPage(transactionId, profiler);
}
pageId = _currentPage;
segmentId = _nextSegment;
for (int i = 0; i < (byteCount + 4); i += _segmentSize)
{
_pageStore.Write(transactionId, _currentPage, resourceBytes, i, _nextSegment * _segmentSize,
_segmentSize < (byteCount + 4 - i) ? _segmentSize : (byteCount + 4 - i),
profiler);
_nextSegment++;
if (_nextSegment == _pointerSegment)
{
StartNewPage(transactionId, profiler);
}
}
}
}
}
public void Delete(ulong txnId, byte[] key, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.Delete"))
{
var root = GetNode(_rootId, profiler);
if (root is ILeafNode)
{
(root as ILeafNode).Delete(txnId, key);
MarkDirty(txnId, root, profiler);
// Update root page pointer - see note in Insert() method above
_rootId = root.PageId;
}
else
{
bool underAllocation;
Delete(txnId, root as IInternalNode, key, out underAllocation, profiler);
if (root.KeyCount == 0)
{
// Now has only a single child leaf node, which should become the new tree root
root = GetNode((root as IInternalNode).GetChildPointer(0), profiler);
_rootId = root.PageId;
}
else
{
// Update root page pointer - see note in Insert() method above
_rootId = root.PageId;
}
}
}
}
/// <summary>
/// Retrieves the resource with the specified resource ID
/// </summary>
/// <param name="resourceId">The resource ID to look for</param>
/// <param name="addToCache">Boolean flag indicating if the returned resource structure should be cached for faster future lookups</param>
/// <param name="profiler">OPTIONAL: Profiler to use to profile this call</param>
/// <returns>The corresponding resource or null if no match is found</returns>
public IResource GetResource(ulong resourceId, bool addToCache, BrightstarProfiler profiler = null)
{
using (profiler.Step("ResourceIndex.GetResource"))
{
IResource resource;
if (_resourceCache.TryGetValue(resourceId, out resource))
{
return(resource);
}
var buff = new byte[64];
if (Search(resourceId, buff, profiler))
{
resource = _resourceStore.FromBTreeValue(buff);
_resourceCache.Add(resourceId, resource);
return(resource);
}
#if DEBUG
if (resourceId > 0)
{
// Repeat the search for debug purposes
Search(resourceId, buff, profiler);
}
#endif
return(null);
}
}
public IEnumerable <KeyValuePair <byte[], byte []> > Scan(BrightstarProfiler profiler)
{
using (profiler.Step("Scan Entire BTree"))
{
return(Scan(_root, profiler));
}
}
public void Write(ulong commitId, ulong pageId, byte[] data, int srcOffset = 0, int pageOffset = 0, int len = -1, BrightstarProfiler profiler = null)
{
if (pageId < _newPageOffset)
{
throw new InvalidOperationException("Attempt to write to a fixed page");
}
var pageIx = (int)(pageId - _newPageOffset);
if (pageIx >= _newPages.Count)
{
throw new InvalidOperationException("Attempt to write to an unreserved page");
}
using (profiler.Step("Write Page"))
{
var page = Retrieve(pageId, profiler);
page.SetData(data, srcOffset, pageOffset, len);
if (_backgroundPageWriter != null)
{
#if DEBUG_PAGESTORE
Logging.LogDebug("Mark {0}", page.Id);
#endif
_backgroundPageWriter.QueueWrite(page, commitId);
}
}
}
/// <summary>
/// Retrieve the resource at the specified page and segment offset
/// </summary>
/// <param name="pageId">The ID of the page that holds the resource to be retrieved</param>
/// <param name="segment">The index of the segment within the page that holds the start of the resource</param>
/// <param name="profiler"></param>
/// <returns>The resource</returns>
public string GetResource(ulong pageId, byte segment, BrightstarProfiler profiler)
{
using (profiler.Step("ResourceTable.GetResource"))
{
var currentPage = _pageStore.Retrieve(pageId, profiler);
int resourceLength = BitConverter.ToInt32(currentPage, segment * _segmentSize);
int totalLength = resourceLength + 4;
int segmentsToLoad = totalLength / _segmentSize;
if (totalLength % _segmentSize > 0)
{
segmentsToLoad++;
}
var buffer = new byte[segmentsToLoad * _segmentSize];
byte segmentIndex = segment;
for (int i = 0; i < segmentsToLoad; i++)
{
if (segmentIndex == _pointerSegment)
{
ulong nextPageId = BitConverter.ToUInt64(currentPage, _pageStore.PageSize - 8);
currentPage = _pageStore.Retrieve(nextPageId, profiler);
segmentIndex = 0;
}
Array.Copy(currentPage, segmentIndex * _segmentSize, buffer, i * _segmentSize, _segmentSize);
segmentIndex++;
}
return(Encoding.UTF8.GetString(buffer, 4, resourceLength));
}
}
private PredicateRelatedResourceIndex AssertPredicateIndex(ulong transactionId, ulong predicateId, BrightstarProfiler profiler)
{
using (profiler.Step("AssertPredicateIndex"))
{
PredicateRelatedResourceIndex predicateIndex = GetPredicateIndex(predicateId, profiler);
if (predicateIndex == null)
{
using (profiler.Step("New Predicate Index"))
{
predicateIndex = new PredicateRelatedResourceIndex(transactionId, PageStore);
_predicateIndexes[predicateId] = predicateIndex;
}
}
return(predicateIndex);
}
}
public void TestSimpleJoinQuery()
{
var sparqlQuery =
@"PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/>
SELECT ?review WHERE {
?review bsbm:reviewFor ?product .
?product bsbm:productFeature <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature2330> .
} LIMIT 3";
// Warm-up
var client = BrightstarService.GetClient("type=embedded;storesDirectory=stores");
for (int i = 0; i < 5; i++)
{
client.ExecuteQuery("SparqlPerformance", sparqlQuery);
}
// Profile
var profiler = new BrightstarProfiler("SimpleJoinQuery");
for (int i = 0; i < 100; i++)
{
using (profiler.Step("ExecuteQuery"))
{
client.ExecuteQuery("SparqlPerformance", sparqlQuery);
}
}
Console.WriteLine(profiler.GetLogString());
}
/// <summary>
/// Commits all changed and new pages to the page store
/// </summary>
/// <param name="commitId">The transaction identifier for the commit</param>
/// <param name="profiler"></param>
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Commit"))
{
using (var outputStream = _persistenceManager.GetOutputStream(_filePath, FileMode.Open))
{
try
{
foreach (var entry in _modifiedPages.OrderBy(e => e.Key))
{
entry.Value.Write(outputStream, commitId);
lock (_pageCacheLock)
{
PageCache.Instance.InsertOrUpdate(_filePath, entry.Value);
}
}
_modifiedPages.Clear();
}
catch (Exception)
{
_modifiedPages.Clear();
throw;
}
}
CurrentTransactionId = commitId;
}
}
public void Triple(string subject, bool subjectIsBNode,
string predicate, bool predicateIsBNode,
string obj, bool objIsBNode,
bool isLiteral, string dataType, string langCode, string graphUri)
{
#if DEBUG_INSERTS
Logging.LogDebug("Triple {0} {1} {2} {3} {4} {5} {6} {7} {8} {9}",
subject, subjectIsBNode,
predicate, predicateIsBNode,
obj, objIsBNode,
isLiteral, dataType, langCode, graphUri);
#endif
using (_profiler.Step("Assert BNodes"))
{
// Convert BNode identifiers to internal BNode identifiers)
if (subjectIsBNode)
{
subject = AssertBNode(subject);
}
if (predicateIsBNode)
{
predicate = AssertBNode(predicate);
}
if (objIsBNode)
{
obj = AssertBNode(obj);
}
}
#if DEBUG_INSERTS
Logging.LogDebug("InsertTriple {0} {1} {2} {3} {4} {5} {6}",
subject, predicate, obj, isLiteral, dataType, langCode, graphUri);
#endif
_store.InsertTriple(subject, predicate, obj, isLiteral, dataType, langCode, graphUri, _profiler);
_total++;
if (_total % _batchSize == 0)
{
if (_commitEachBatch)
{
Logging.LogInfo("Committing...");
_store.Commit(_jobId, _profiler);
}
#if MONITOR_MEMORY_LOAD
else
{
if (Server.MemoryUtils.GetMemoryLoad() > HighMemoryLoad)
{
Logging.LogInfo("Flushing...");
_store.FlushChanges(_profiler);
// Clear out any interned URIs to free up more memory
VDS.RDF.UriFactory.Clear();
Logging.LogDebug("Initiating garbage collection...");
GC.Collect();
}
}
#endif
Logging.LogInfo("Complete @ " + _total);
}
}
public virtual ulong Save(ulong transactionId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.Save"))
{
_isDirty = false;
return(RootId);
}
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
using (profiler.Step("RelatedResourceIndex.Write"))
{
var indexBuilder = new BPlusTreeBuilder(pageStore, Configuration);
return(indexBuilder.Build(transactionId, WritePredicateIndexes(pageStore, transactionId, profiler),
profiler));
}
}
/// <summary>
/// Finds or creates a new ID for the graph with the specified graph URI
/// </summary>
/// <param name="graphUri">The graph URI to lookup</param>
/// <param name="profiler"></param>
/// <returns>The ID assigned to the graph</returns>
public int AssertGraphId(string graphUri, BrightstarProfiler profiler = null)
{
if (String.IsNullOrEmpty(graphUri))
{
throw new ArgumentException("Graph URI must not be null or an empty string", "graphUri");
}
if (graphUri.Length > short.MaxValue)
{
throw new ArgumentException(
String.Format("Graph URI string exceeds maximum allowed length of {0} bytes", short.MaxValue), "graphUri");
}
#if WINDOWS_PHONE || PORTABLE
lock (_lock)
{
int entryId;
if (_graphUriIndex.TryGetValue(graphUri, out entryId) && !_allEntries[entryId].IsDeleted)
{
return(entryId);
}
var newId = _allEntries.Count;
var entry = new GraphIndexEntry(newId, graphUri, false);
_allEntries.Add(entry);
_graphUriIndex.Add(graphUri, newId);
return(newId);
}
#else
using (profiler.Step("Assert Graph Id"))
{
_lock.EnterUpgradeableReadLock();
try
{
int entryId;
if (_graphUriIndex.TryGetValue(graphUri, out entryId) && !_allEntries[entryId].IsDeleted)
{
return(entryId);
}
_lock.EnterWriteLock();
try
{
var newId = _allEntries.Count;
var entry = new GraphIndexEntry(newId, graphUri, false);
_allEntries.Add(entry);
_graphUriIndex.Add(graphUri, newId);
return(newId);
}
finally
{
_lock.ExitWriteLock();
}
}
finally
{
_lock.ExitUpgradeableReadLock();
}
}
#endif
}
/// <summary>
/// Adds a related resource index entry
/// </summary>
/// <param name="txnId"> </param>
/// <param name="resourceId">The resource ID of the "start" resource</param>
/// <param name="predicateId">The resource ID of the predicate that relates the resources</param>
/// <param name="relatedResourceId">The resource ID of the "related" resource</param>
/// <param name="graphId">The resource ID of the graph containing the relationship</param>
/// <param name="profiler"></param>
public void AddRelatedResource(ulong txnId, ulong resourceId, ulong predicateId, ulong relatedResourceId, int graphId, BrightstarProfiler profiler = null)
{
using (profiler.Step("Add Related Resource"))
{
var predicateIndex = AssertPredicateIndex(txnId, predicateId, profiler);
var key = MakePredicateIndexKey(resourceId, graphId, relatedResourceId);
try
{
using (profiler.Step("Insert Into Predicate Index"))
{
predicateIndex.Insert(txnId, key, null, profiler: profiler);
}
}
catch (DuplicateKeyException)
{
// Ignore duplicate key exceptions
}
}
}
/// <summary>
/// Adds a related resource index entry
/// </summary>
/// <param name="txnId"> </param>
/// <param name="resourceId">The resource ID of the "start" resource</param>
/// <param name="predicateId">The resource ID of the predicate that relates the resources</param>
/// <param name="relatedResourceId">The resource ID of the "related" resource</param>
/// <param name="graphId">The resource ID of the graph containing the relationship</param>
/// <param name="profiler"></param>
public void AddRelatedResource(ulong txnId, ulong resourceId, ulong predicateId, ulong relatedResourceId, int graphId, BrightstarProfiler profiler = null)
{
using (profiler.Step("Add Related Resource"))
{
var predicateIndex = AssertPredicateIndex(txnId, predicateId, profiler);
var key = MakePredicateIndexKey(resourceId, graphId, relatedResourceId);
try
{
using (profiler.Step("Insert Into Predicate Index"))
{
predicateIndex.Insert(txnId, key, null, profiler: profiler);
}
}
catch (DuplicateKeyException)
{
// Ignore duplicate key exceptions
}
}
}
public ConcurrentGraphIndex(IPageStore pageStore, ulong rootPage, BrightstarProfiler profiler)
{
using (profiler.Step("Load ConcurrentGraphIndex"))
{
_pageStore = pageStore;
_graphUriIndex = new Dictionary <string, int>();
_allEntries = new List <GraphIndexEntry>();
Read(rootPage, profiler);
}
}
public static IPageStore CreateEmptyPageStore(string fileName, PersistenceType persistenceType = PersistenceType.AppendOnly, BrightstarProfiler profiler = null)
{
using (profiler.Step("Create Empty Page Store"))
{
using (profiler.Step("Delete Existing"))
{
if (PersistenceManager.FileExists(fileName)) PersistenceManager.DeleteFile(fileName);
//if (File.Exists(fileName)) File.Delete(fileName);
}
using (profiler.Step("Create New Page Store"))
{
if (persistenceType == PersistenceType.AppendOnly)
{
return new AppendOnlyFilePageStore(PersistenceManager, fileName, 4096, false, false);
}
return new BinaryFilePageStore(PersistenceManager, fileName, 4096, false, 1);
}
}
}
public ConcurrentGraphIndex(IPageStore pageStore, ulong rootPage, BrightstarProfiler profiler)
{
using (profiler.Step("Load ConcurrentGraphIndex"))
{
_pageStore = pageStore;
_graphUriIndex = new Dictionary<string, int>();
_allEntries = new List<GraphIndexEntry>();
Read(rootPage, profiler);
}
}
/// <summary>
/// Writes data to the specified page
/// </summary>
/// <param name="commitId">The transaction id for the update</param>
/// <param name="pageId">The ID of the page to write to</param>
/// <param name="buff">The data to be written</param>
/// <param name="srcOffset">The offset into <paramref name="buff"/> from which to start copying bytes. Defaults to 0</param>
/// <param name="pageOffset">The offset into the page data buffer to start writing to. Defaults to 0</param>
/// <param name="len">The number of bytes to write. Defaults to all bytes in <paramref name="buff"/> from the specified <paramref name="srcOffset"/></param>
/// <param name="profiler"></param>
public void Write(ulong commitId, ulong pageId, byte[] buff, int srcOffset = 0, int pageOffset = 0, int len = -1, BrightstarProfiler profiler = null)
{
using (profiler.Step("Write Page"))
{
var page = GetPage(pageId, profiler);
var writeBuff = page.GetWriteBuffer(commitId);
Array.Copy(buff, srcOffset, writeBuff, pageOffset, len < 0 ? buff.Length : len);
_modifiedPages[pageId] = new Tuple <BinaryFilePage, ulong>(page, commitId);
}
}
public Store(string storeLocation, IPageStore dataPageStore, IResourceTable resourceTable, ulong storePageId, BrightstarProfiler profiler)
{
using (profiler.Step("Load Store"))
{
DirectoryPath = storeLocation;
_pageStore = dataPageStore;
_resourceTable = resourceTable;
var storePage = _pageStore.Retrieve(storePageId, profiler);
Load(storePage, profiler);
}
}
public bool ContainsRelatedResource(ulong resourceId, ulong predicateId, ulong relatedResourceId, int graphId, BrightstarProfiler profiler)
{
using (profiler.Step("ContainsRelatedResource"))
{
byte[] valueBuff = null;
PredicateRelatedResourceIndex predicateIndex = GetPredicateIndex(predicateId, profiler);
return(predicateIndex != null &&
predicateIndex.Search(MakePredicateIndexKey(resourceId, graphId, relatedResourceId), valueBuff,
profiler));
}
}
/// <summary>
/// Enumerates the key-value pairs stored in the BTree starting with <paramref name="fromKey"/>
/// up to <paramref name="toKey"/> (inclusive)
/// </summary>
/// <param name="fromKey">The lowest key to return in the enumeration</param>
/// <param name="toKey">The highest key to return in the enumeration</param>
/// <param name="profiler"></param>
/// <returns>An enumeration of key-value pairs from the BTree</returns>
public IEnumerable <KeyValuePair <byte[], byte[]> > Scan(byte[] fromKey, byte[] toKey, BrightstarProfiler profiler)
{
using (profiler.Step("Scan BTree Range"))
{
if (fromKey.Compare(toKey) > 1)
{
throw new ArgumentException("Scan can only be performed in increasing order.");
}
return(Scan(_root, fromKey, toKey, profiler));
}
}
public static IPageStore OpenPageStore(string fileName, bool readOnly, PersistenceType persistenceType = PersistenceType.AppendOnly, ulong txnId = 1UL, BrightstarProfiler profiler = null)
{
using (profiler.Step("Open Page Store"))
{
if (persistenceType == PersistenceType.AppendOnly)
{
return new AppendOnlyFilePageStore(PersistenceManager, fileName, 4096, readOnly, false);
}
return new BinaryFilePageStore(PersistenceManager, fileName, 4096, readOnly, txnId);
}
}
public Store(string storeLocation, IPageStore dataPageStore, IResourceTable resourceTable, ulong storePageId, BrightstarProfiler profiler)
{
using (profiler.Step("Load Store"))
{
DirectoryPath = storeLocation;
_pageStore = dataPageStore;
_resourceTable = resourceTable;
var storePage = _pageStore.Retrieve(storePageId, profiler);
Load(storePage, profiler);
}
}
/// <summary>
/// Enumerates the resources related to a start resource by a specific predicate
/// </summary>
/// <param name="resourceId">The resource ID of the start resource</param>
/// <param name="predicateId">The resource ID of the predicate that relates the resources. If set to Constants.NullUlong, returns relationships for all predicates</param>
/// <param name="graphId">The resource ID of the graph containing the relationships. If set to Constants.NullUlong, returns relationships from all graphs</param>
/// <param name="profiler"></param>
/// <returns></returns>
public IEnumerable<IRelatedResource> EnumerateRelatedResources(ulong resourceId, ulong predicateId = StoreConstants.NullUlong, int graphId = -1, BrightstarProfiler profiler = null)
{
using (profiler.Step("EnumerateRelatedResources"))
{
if (predicateId != StoreConstants.NullUlong)
{
byte[] minKey = MakePredicateIndexKey(resourceId,
graphId < 0 ? 0 : graphId,
UInt64.MinValue);
byte[] maxKey = MakePredicateIndexKey(resourceId,
graphId < 0 ? Int32.MaxValue : graphId,
UInt64.MaxValue);
var predicateIndex = GetPredicateIndex(predicateId, profiler);
if (predicateIndex != null)
{
foreach (var r in predicateIndex.Scan(minKey, maxKey, profiler).Select(
x =>
new RelatedResource(predicateId, GetGraphIdFromKey(x.Key),
GetRelatedResourceIdFromKey(x.Key))))
{
yield return r;
}
}
}
else
{
foreach (var entry in Scan(0ul, UInt64.MaxValue, profiler))
{
// Load the predicate index into the cache
GetPredicateIndex(entry.Key,BitConverter.ToUInt64(entry.Value, 0), profiler);
// Then a recursive call to enumerate the index tree
foreach (var r in EnumerateRelatedResources(resourceId, entry.Key, graphId, profiler))
{
yield return r;
}
}
}
}
}
/// <summary>
/// Ensures that the specified resource is in the resource index and returns its resource ID
/// </summary>
/// <param name="txnId">The ID of the current update transaction</param>
/// <param name="resourceValue">The resource string value</param>
/// <param name="isLiteral">Boolean flag indicating if the resource is a literal (true) or a URI (false)</param>
/// <param name="dataType">The resource data-type URI</param>
/// <param name="langCode">The resource language string</param>
/// <param name="addToCache">Boolean flag indicating if newly indexed resources should be added to the local cache</param>
/// <param name="profiler"></param>
/// <returns>The resource ID for the resource</returns>
public ulong AssertResourceInIndex(ulong txnId, string resourceValue, bool isLiteral = false, string dataType = null, string langCode = null, bool addToCache = true, BrightstarProfiler profiler = null)
{
using (profiler.Step("AssertResourceInIndex"))
{
// Normalize language code to null if it is an empty string
if (String.IsNullOrEmpty(langCode))
{
langCode = null;
}
// Retrieve the resource ID for the datatype URI (if any)
var dataTypeId = String.IsNullOrEmpty(dataType)
? StoreConstants.NullUlong
: AssertResourceInIndex(txnId, dataType, profiler:profiler);
var hashString = isLiteral ? MakeHashString(resourceValue, dataType, langCode) : resourceValue;
ulong resourceId;
if (_resourceIdCache.TryGetValue(hashString, out resourceId))
{
return resourceId;
}
// Get a ulong resource ID for the language code string
var langCodeId = String.IsNullOrEmpty(langCode)
? StoreConstants.NullUlong
: AssertResourceInIndex(txnId, langCode, true, profiler:profiler);
resourceId = AssertResourceInBTree(txnId, resourceValue, isLiteral, dataTypeId, langCodeId,
StringExtensions.GetBrightstarHashCode(hashString), profiler);
if (addToCache)
{
_resourceIdCache.Add(hashString, resourceId);
}
return resourceId;
}
}
/*
* Data format for the store page:
*
* 00-03: Store format version (int)
* 04-11: Transaction id (ulong)
* 12-19: Graph Index start page id (ulong)
* 20-27: Prefix Manager start page id (ulong)
* 28-35: Resource Index start page id (ulong)
* 36-43: Subject Related Resource Index start page id (ulong)
* 44-51: Object Related Resource Index start page id (ulong)
* 44-107: Reserved (all zeros for now)
* 108-127: SHA1 hash of bytes 00-108
* 128-255: Repeat of the above structure
*/
private bool Load(IPage storePage, BrightstarProfiler profiler)
{
using (profiler.Step("Store.Load"))
{
// Validate the hash for the index bloc
using (var sha1 = new SHA1Managed())
{
var recordedHash = new byte[20];
Array.Copy(storePage.Data, 108, recordedHash, 0, 20);
var calculatedHash = sha1.ComputeHash(storePage.Data, 0, 108);
if (recordedHash.Compare(calculatedHash) != 0)
{
return false;
}
}
// Load indexes from the pointers
int storeVersion = BitConverter.ToInt32(storePage.Data, 0);
if (storeVersion == 1)
{
_currentTxnId = BitConverter.ToUInt64(storePage.Data, 4);
var graphIndexId = BitConverter.ToUInt64(storePage.Data, 12);
_graphIndex = new ConcurrentGraphIndex(_pageStore, graphIndexId, profiler);
var prefixManagerId = BitConverter.ToUInt64(storePage.Data, 20);
_prefixManager = new PrefixManager(_pageStore, prefixManagerId, profiler);
var resourceIndexId = BitConverter.ToUInt64(storePage.Data, 28);
_resourceIndex = new ResourceIndex.ResourceIndex(_pageStore, _resourceTable, resourceIndexId);
var relatedResourceIndexId = BitConverter.ToUInt64(storePage.Data, 36);
_subjectRelatedResourceIndex = new RelatedResourceIndex.RelatedResourceIndex(_pageStore,
relatedResourceIndexId, profiler);
var objectRelatedResourceIndexId = BitConverter.ToUInt64(storePage.Data, 44);
_objectRelatedResourceIndex = new RelatedResourceIndex.RelatedResourceIndex(_pageStore,
objectRelatedResourceIndexId, profiler);
}
return true;
}
}
private ulong Insert(ulong txnId, INode node, byte[] key, byte[] value, out bool split, out INode rightNode, out byte[] splitKey, bool overwrite, BrightstarProfiler profiler)
{
if (node is LeafNode)
{
var leaf = node as LeafNode;
if (leaf.IsFull)
{
var newNode = leaf.Split(_pageStore.Create(), out splitKey);
_modifiedNodes[newNode.PageId] = newNode;
if (key.Compare(splitKey) < 0)
{
leaf.Insert(key, value, overwrite, profiler);
}
else
{
newNode.Insert(key, value, overwrite, profiler);
}
MarkDirty(txnId, leaf, profiler);
split = true;
rightNode = newNode;
}
else
{
leaf.Insert(key, value, overwrite, profiler);
MarkDirty(txnId, leaf, profiler);
split = false;
rightNode = null;
splitKey = null;
}
return leaf.PageId;
}
else
{
var internalNode = node as InternalNode;
var childNodeId = internalNode.GetChildNodeId(key);
var childNode = GetNode(childNodeId, profiler);
bool childSplit;
INode rightChild;
byte[] childSplitKey;
var newChildNodeId = Insert(txnId, childNode, key, value, out childSplit, out rightChild, out childSplitKey, overwrite, profiler);
if (childSplit)
{
if (internalNode.IsFull)
{
using (profiler.Step("Split Internal Node"))
{
// Need to split this node to insert the new child node
rightNode = internalNode.Split(_pageStore.Create(), out splitKey);
_modifiedNodes[rightNode.PageId] = rightNode;
split = true;
if (childSplitKey.Compare(splitKey) < 0)
{
internalNode.Insert(childSplitKey, rightChild.PageId);
}
else
{
(rightNode as InternalNode).Insert(childSplitKey, rightChild.PageId);
}
// update child pointers if required (need to check both internalNode and rightNode as we don't know which side the modified child node ended up on)
if (newChildNodeId != childNodeId)
{
internalNode.UpdateChildPointer(childNodeId, newChildNodeId);
(rightNode as InternalNode).UpdateChildPointer(childNodeId, newChildNodeId);
}
}
}
else
{
using (profiler.Step("Insert into internal node"))
{
split = false;
rightNode = null;
splitKey = null;
internalNode.Insert(childSplitKey, rightChild.PageId);
}
}
using (profiler.Step("Update Child Pointer"))
{
if (newChildNodeId != childNodeId)
{
internalNode.UpdateChildPointer(childNodeId, newChildNodeId);
}
MarkDirty(txnId, internalNode, profiler);
}
return internalNode.PageId;
}
else
{
using (profiler.Step("Update Child Pointer"))
{
if (newChildNodeId != childNodeId)
{
internalNode.UpdateChildPointer(childNodeId, newChildNodeId);
MarkDirty(txnId, internalNode, profiler);
}
split = false;
rightNode = null;
splitKey = null;
return internalNode.PageId;
}
}
}
}
public void Commit(Guid jobId, BrightstarProfiler profiler = null)
{
using (profiler.Step("Store Commit"))
{
ulong storePageId = Save(profiler);
var storeManager = StoreManagerFactory.GetStoreManager();
var mf = storeManager.GetMasterFile(DirectoryPath);
mf.AppendCommitPoint(new CommitPoint(storePageId,_currentTxnId + 1,DateTime.UtcNow, jobId));
_currentTxnId++;
}
}
/// <summary>
/// Insert a triple into the store
/// </summary>
/// <param name="subject"></param>
/// <param name="predicate"></param>
/// <param name="objValue"></param>
/// <param name="isObjectLiteral"></param>
/// <param name="dataType"></param>
/// <param name="langCode"></param>
/// <param name="graphUri"></param>
/// <param name="profiler"></param>
public void InsertTriple(string subject, string predicate, string objValue, bool isObjectLiteral, string dataType, string langCode, string graphUri, BrightstarProfiler profiler = null)
{
using (profiler.Step("InsertTriple"))
{
using (profiler.Step("Normalization"))
{
// Normalize subject, predicate, objValue (if it is not a literal), dataType (if it is not null)
// and graphUri (if it is not null)
try
{
subject = new Uri(subject, UriKind.Absolute).ToString();
}
catch (FormatException)
{
throw new InvalidTripleException(
String.Format("The subject '{0}' could not be parsed as a valid URI.", subject));
}
try
{
predicate = new Uri(predicate, UriKind.Absolute).ToString();
}
catch (FormatException)
{
throw new InvalidTripleException(
String.Format("The predicate'{0}' could not be parsed as a valid URI.", predicate));
}
if (!isObjectLiteral)
{
try
{
objValue = new Uri(objValue, UriKind.Absolute).ToString();
}
catch (FormatException)
{
throw new InvalidTripleException(
String.Format("The object '{0}' could not be parsed as a valid URI.", objValue));
}
}
if (isObjectLiteral && !String.IsNullOrEmpty(dataType))
{
try
{
dataType = new Uri(dataType, UriKind.Absolute).ToString();
}
catch (FormatException)
{
throw new InvalidTripleException(
String.Format("The dataType '{0}' could not be parsed as a valid URI.", dataType));
}
}
if (!String.IsNullOrEmpty(graphUri))
{
try
{
graphUri = new Uri(graphUri, UriKind.Absolute).ToString();
}
catch (FormatException)
{
throw new InvalidTripleException(
String.Format("The graphUri '{0}' could not be parsed as a valid URI.", graphUri));
}
}
if (isObjectLiteral && dataType == null)
{
dataType = RdfDatatypes.PlainLiteral;
}
// Normalize language code to lower-case (per http://www.w3.org/TR/rdf-concepts/#section-Graph-Literal)
if (langCode != null)
{
langCode = langCode.ToLowerInvariant();
}
}
using (profiler.Step("Make Prefixed Uris"))
{
subject = _prefixManager.MakePrefixedUri(subject);
predicate = _prefixManager.MakePrefixedUri(predicate);
if (!isObjectLiteral)
{
objValue = _prefixManager.MakePrefixedUri(objValue);
}
}
var txnId = _currentTxnId + 1;
ulong sid = _resourceIndex.AssertResourceInIndex(txnId, subject, profiler:profiler);
ulong pid = _resourceIndex.AssertResourceInIndex(txnId, predicate, profiler:profiler);
ulong oid = _resourceIndex.AssertResourceInIndex(txnId, objValue, isObjectLiteral, dataType, langCode,
!isObjectLiteral, profiler);
int gid = _graphIndex.AssertGraphId(graphUri, profiler);
_subjectRelatedResourceIndex.AddRelatedResource(txnId, sid, pid, oid, gid, profiler);
_objectRelatedResourceIndex.AddRelatedResource(txnId, oid, pid, sid, gid, profiler);
}
}
public void Commit(ulong transactionId, BrightstarProfiler profiler)
{
using (profiler.Step("Commit ResourceTable"))
{
if (_currentPage > 0)
{
_pageStore.Commit(transactionId, profiler);
_nextSegment = _pointerSegment;
_currentPage = 0;
}
}
}
/// <summary>
/// Enumerates the key-value pairs stored in the BTree starting with <paramref name="fromKey"/>
/// up to <paramref name="toKey"/> (inclusive)
/// </summary>
/// <param name="fromKey">The lowest key to return in the enumeration</param>
/// <param name="toKey">The highest key to return in the enumeration</param>
/// <param name="profiler"></param>
/// <returns>An enumeration of key-value pairs from the BTree</returns>
public IEnumerable<KeyValuePair<byte[], byte[]>> Scan(byte[] fromKey, byte[] toKey, BrightstarProfiler profiler )
{
using (profiler.Step("Scan BTree Range"))
{
if (fromKey.Compare(toKey) > 1)
{
throw new ArgumentException("Scan can only be performed in increasing order.");
}
return Scan(_root, fromKey, toKey, profiler);
}
}
public void Insert(ulong txnId, byte[] key, byte[] value, bool overwrite = false, BrightstarProfiler profiler = null)
{
using (profiler.Step("BPlusTree.Insert"))
{
bool splitRoot;
INode rightNode;
byte[] rootSplitKey;
Insert(txnId, _root, key, value, out splitRoot, out rightNode, out rootSplitKey, overwrite, profiler);
if (splitRoot)
{
var newRoot = new InternalNode(_pageStore.Create(), rootSplitKey, _root.PageId, rightNode.PageId,
_config);
MarkDirty(txnId, _root, profiler);
_modifiedNodes[newRoot.PageId] = newRoot;
_root = newRoot;
}
}
}
public void Insert(ulong transactionId, string resource, out ulong pageId, out byte segmentId, BrightstarProfiler profiler)
{
using (profiler.Step("ResourceTable.Insert"))
{
var byteCount = Encoding.UTF8.GetByteCount(resource);
var resourceBytes = new byte[byteCount + 4];
BitConverter.GetBytes(byteCount).CopyTo(resourceBytes, 0);
Encoding.UTF8.GetBytes(resource, 0, resource.Length, resourceBytes, 4);
lock (_writeLock)
{
if (_nextSegment == _pointerSegment)
{
StartNewPage(transactionId, profiler);
}
pageId = _currentPage;
segmentId = _nextSegment;
for (int i = 0; i < (byteCount + 4); i += _segmentSize)
{
_pageStore.Write(transactionId, _currentPage, resourceBytes, i, _nextSegment*_segmentSize,
_segmentSize < (byteCount + 4 - i) ? _segmentSize : (byteCount + 4 - i),
profiler);
_nextSegment++;
if (_nextSegment == _pointerSegment)
{
StartNewPage(transactionId, profiler);
}
}
}
}
}
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Commit"))
{
var pagesToWrite = _newPages.Where(rf => rf.IsAlive).Select(rf => rf.Target as IPage).Where(pg => pg != null);
if (_backgroundPageWriter != null)
{
foreach (var page in pagesToWrite)
{
_backgroundPageWriter.QueueWrite(page, commitId);
}
_backgroundPageWriter.Flush();
RestartBackgroundWriter();
}
else
{
using (var outputStream = _peristenceManager.GetOutputStream(_path, FileMode.Open))
{
foreach (var page in pagesToWrite)
{
page.Write(outputStream, commitId);
}
}
}
_newPages.Clear();
_newPageOffset = _nextPageId;
PageCache.Instance.Clear(_path);
}
}
public virtual ulong Save(ulong transactionId, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.Save"))
{
foreach (var n in _modifiedNodes.Values)
{
_pageStore.Write(transactionId, n.PageId, n.GetData(), profiler: profiler);
n.IsDirty = false;
//_nodeCache.Add(n.PageId, new WeakReference(n));
_nodeCache.Add(n);
}
_modifiedNodes.Clear();
return RootId;
}
}
/// <summary>
/// Performs the actual load of prefixes from a page.
/// </summary>
/// <param name="page"></param>
/// <param name="profiler"></param>
/// <remarks>Calls to this method should be made inside a critical section of code protected with a mutex or reader/writer lock</remarks>
private void InterlockedLoad(byte[] page, BrightstarProfiler profiler)
{
using (profiler.Step("PrefixManager.InterlockedLoad"))
{
int offset = 0;
while (offset < _pageStore.PageSize)
{
ushort prefixLength = BitConverter.ToUInt16(page, offset);
offset += 2;
if (prefixLength == ushort.MaxValue)
{
ulong nextPageId = BitConverter.ToUInt64(page, offset);
if (nextPageId == 0)
{
// End of data
return;
}
page = _pageStore.Retrieve(nextPageId, profiler);
offset = 0;
}
else
{
var prefix = Encoding.UTF8.GetString(page, offset, prefixLength);
offset += prefixLength;
var uriLen = BitConverter.ToUInt16(page, offset);
offset += 2;
var uri = Encoding.UTF8.GetString(page, offset, uriLen);
offset += uriLen;
_prefixMappings[uri] = prefix;
_shortValueMappings[prefix] = uri;
}
}
}
}
public bool Search(byte[] key, byte[] valueBuff, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.Search"))
{
INode u = _root;
while (u is InternalNode)
{
var internalNode = u as InternalNode;
u = GetNode(internalNode.GetChildNodeId(key), profiler);
}
var l = u as LeafNode;
return l.GetValue(key, valueBuff);
}
}
public void Delete(ulong txnId, byte[] key, BrightstarProfiler profiler)
{
using (profiler.Step("BPlusTree.Delete"))
{
if (_root is LeafNode)
{
(_root as LeafNode).Delete(key);
MarkDirty(txnId, _root, profiler);
}
else
{
bool underAllocation;
Delete(txnId, _root as InternalNode, key, out underAllocation, profiler);
if (_root.KeyCount ==0)
{
// Now has only a single child leaf node, which should become the new tree root
_root = GetNode((_root as InternalNode).ChildPointers[0], profiler);
}
}
}
}
private ulong Save(BrightstarProfiler profiler)
{
using (profiler.Step("Store.Save"))
{
_resourceTable.Commit(_currentTxnId + 1, profiler);
var txnId = _currentTxnId + 1;
var graphIndexId = _graphIndex.Save(txnId, profiler);
var prefixManagerId = _prefixManager.Save(txnId, profiler);
var resourceIndexId = _resourceIndex.Save(txnId, profiler);
var subjectRelatedResourceIndexId = _subjectRelatedResourceIndex.Save(txnId, profiler);
var objectRelatedResourceIndexId = _objectRelatedResourceIndex.Save(txnId, profiler);
var buff = CreateStoreHeader(graphIndexId, prefixManagerId, resourceIndexId,
subjectRelatedResourceIndexId, objectRelatedResourceIndexId);
var page = _pageStore.Create(txnId);
page.SetData(buff);
page.SetData(buff, 0, 128);
_pageStore.Commit(txnId, profiler);
return page.Id;
}
}
public void Write(ulong commitId, ulong pageId, byte[] data, int srcOffset = 0, int pageOffset = 0, int len = -1, BrightstarProfiler profiler = null)
{
if (pageId < _newPageOffset)
{
throw new InvalidOperationException("Attempt to write to a fixed page");
}
var pageIx = (int) (pageId - _newPageOffset);
if (pageIx >= _newPages.Count)
{
throw new InvalidOperationException("Attempt to write to an unreserved page");
}
using (profiler.Step("Write Page"))
{
var page = Retrieve(pageId, profiler);
page.SetData(data, srcOffset, pageOffset, len);
}
}
/// <summary>
/// Finds or creates a new ID for the graph with the specified graph URI
/// </summary>
/// <param name="graphUri">The graph URI to lookup</param>
/// <param name="profiler"></param>
/// <returns>The ID assigned to the graph</returns>
public int AssertGraphId(string graphUri, BrightstarProfiler profiler = null)
{
if (String.IsNullOrEmpty(graphUri))
{
throw new ArgumentException("Graph URI must not be null or an empty string", "graphUri");
}
if (graphUri.Length > short.MaxValue)
{
throw new ArgumentException(
String.Format("Graph URI string exceeds maximum allowed length of {0} bytes", short.MaxValue), "graphUri");
}
#if WINDOWS_PHONE
lock(_lock)
{
int entryId;
if (_graphUriIndex.TryGetValue(graphUri, out entryId) && !_allEntries[entryId].IsDeleted)
{
return entryId;
}
var newId = _allEntries.Count;
var entry = new GraphIndexEntry(newId, graphUri, false);
_allEntries.Add(entry);
_graphUriIndex.Add(graphUri, newId);
return newId;
}
#else
using (profiler.Step("Assert Graph Id"))
{
_lock.EnterUpgradeableReadLock();
try
{
int entryId;
if (_graphUriIndex.TryGetValue(graphUri, out entryId) && !_allEntries[entryId].IsDeleted)
{
return entryId;
}
_lock.EnterWriteLock();
try
{
var newId = _allEntries.Count;
var entry = new GraphIndexEntry(newId, graphUri, false);
_allEntries.Add(entry);
_graphUriIndex.Add(graphUri, newId);
return newId;
}
finally
{
_lock.ExitWriteLock();
}
}
finally
{
_lock.ExitUpgradeableReadLock();
}
}
#endif
}
public IEnumerable<KeyValuePair<byte[], byte []>> Scan(BrightstarProfiler profiler)
{
using (profiler.Step("Scan Entire BTree"))
{
return Scan(_root, profiler);
}
}
private void MarkDirty(ulong txnId, INode node, BrightstarProfiler profiler)
{
using (profiler.Step("MarkDirty"))
{
if (!node.IsDirty)
{
//_nodeCache.Remove(node.PageId);
_nodeCache.Remove(node);
if (!_pageStore.IsWriteable(node.PageId))
{
node.PageId = _pageStore.Create();
}
node.IsDirty = true;
_modifiedNodes[node.PageId] = node;
}
_pageStore.Write(txnId, node.PageId, node.GetData(), profiler: profiler);
//Task.Factory.StartNew(() => _pageStore.Write(txnId, node.PageId, node.GetData(), profiler:null)); // Not passing through the profiler because it is not thread-safe
}
}
