public IPage Retrieve(ulong pageId, BrightstarProfiler profiler) { using (profiler.Step("BinaryFilePageStore.GetPage")) { // Look in the page cache IPage page = PageCache.Instance.Lookup(_partitionId, pageId) as BinaryFilePage; if (page != null) { profiler.Incr("PageCache Hit"); return(page); } // See if the page is queued for writing if (_backgroundPageWriter != null && _backgroundPageWriter.TryGetPage(pageId, out page)) { profiler.Incr("BackgroundWriter Queue Hit"); return(page); } // Not found in memory, so go to the disk profiler.Incr("PageCache Miss"); using (profiler.Step("Load Page")) { page = _modifiedPages.ContainsKey(pageId) ? new BinaryFilePage(_inputStream, pageId, _nominalPageSize, _writeTxnId, true) : new BinaryFilePage(_inputStream, pageId, _nominalPageSize, _readTxnId, false); PageCache.Instance.InsertOrUpdate(_partitionId, page); return(page); } } }
public byte[] Retrieve(ulong pageId, BrightstarProfiler profiler) { using (profiler.Step("PageStore.Retrieve")) { if (!_readonly && pageId >= _newPageOffset) { var newPage = _newPages[(int) (pageId - _newPageOffset)]; return newPage.Data; } var page = PageCache.Instance.Lookup(_path, pageId) as FilePage; if (page != null) { profiler.Incr("PageCache Hit"); return page.Data; } using (profiler.Step("Load Page")) { profiler.Incr("PageCache Miss"); using (profiler.Step("Create FilePage")) { // Lock on stream to prevent attempts to concurrently load a page lock (_stream) { page = new FilePage(_stream, pageId, _pageSize); } } using (profiler.Step("Add FilePage To Cache")) { PageCache.Instance.InsertOrUpdate(_path, page); } return page.Data; } } }
private BinaryFilePage GetPage(ulong pageId, BrightstarProfiler profiler) { using (profiler.Step("PageStore.GetPage")) { lock (_pageCacheLock) { BinaryFilePage page; Tuple <BinaryFilePage, ulong> modifiedPage; if (_modifiedPages.TryGetValue(pageId, out modifiedPage)) { profiler.Incr("PageCache Hit"); return(modifiedPage.Item1); } page = PageCache.Instance.Lookup(_filePath, pageId) as BinaryFilePage; if (page != null) { profiler.Incr("PageCache Hit"); return(page); } using (profiler.Step("Load Page")) { profiler.Incr("PageCache Miss"); page = new BinaryFilePage(_inputStream, pageId, _nominalPageSize); PageCache.Instance.InsertOrUpdate(_filePath, page); } return(page); } } }
public INode GetNode(ulong nodeId, BrightstarProfiler profiler) { using (profiler.Step("BPlusTree.GetNode")) { INode ret; if (_nodeCache.TryGetValue(nodeId, out ret)) { profiler.Incr("NodeCache Hit"); return ret; } profiler.Incr("NodeCache Miss"); using (profiler.Step("Load Node")) { var nodePage = _pageStore.Retrieve(nodeId, profiler); var header = BitConverter.ToInt32(nodePage.Data, 0); if (header < 0) { ret = MakeInternalNode(nodePage, ~header); #if DEBUG_BTREE _config.BTreeDebug("{0}: Loaded INTERNAL node from page {1}. {2}",_config.DebugId, nodePage.Id, ret.ToString()); #endif } else { ret = MakeLeafNode(nodePage, header); #if DEBUG_BTREE _config.BTreeDebug("{0}: Loaded LEAF node from page {1}. {2}", _config.DebugId, nodePage.Id, ret.ToString()); #endif } _nodeCache.Add(ret); return ret; } } }
public byte[] Retrieve(ulong pageId, BrightstarProfiler profiler) { using (profiler.Step("PageStore.Retrieve")) { if (!_readonly && pageId >= _newPageOffset) { var newPage = _newPages[(int)(pageId - _newPageOffset)]; return(newPage.Data); } var page = PageCache.Instance.Lookup(_path, pageId) as FilePage; if (page != null) { profiler.Incr("PageCache Hit"); return(page.Data); } using (profiler.Step("Load Page")) { profiler.Incr("PageCache Miss"); using (profiler.Step("Create FilePage")) { // Lock on stream to prevent attempts to concurrently load a page lock (_stream) { page = new FilePage(_stream, pageId, _pageSize); } } using (profiler.Step("Add FilePage To Cache")) { PageCache.Instance.InsertOrUpdate(_path, page); } return(page.Data); } } }
//private static ulong _targetResourceId = 15806097072303636481; private ulong AssertResourceInBTree(ulong txnId, string resourceValue, bool isLiteral, ulong dataTypeId, ulong langCodeId, uint hashCode, BrightstarProfiler profiler) { using (profiler.Step("AssertResourceInBTree")) { ulong rangeMin = MakeId(hashCode, 0), rangeMax = MakeId(hashCode, UInt32.MaxValue); ulong lastResourceId = StoreConstants.NullUlong; foreach (var indexEntry in this.Scan(rangeMin, rangeMax, profiler)) { var resource = _resourceStore.FromBTreeValue(indexEntry.Value); if (resource.Matches(resourceValue, isLiteral, dataTypeId, langCodeId)) { return(indexEntry.Key); } lastResourceId = indexEntry.Key; } // If we reach here, there has been no match so we need to insert a new resource using (profiler.Step("Add Resource To BTree")) { var newResource = _resourceStore.CreateNew(txnId, resourceValue, isLiteral, dataTypeId, langCodeId, profiler); var resourceKey = MakeId(hashCode, GetBucketPosition(lastResourceId) + 1); Insert(txnId, resourceKey, newResource.GetData()); return(resourceKey); } } }
public INode GetNode(ulong nodeId, BrightstarProfiler profiler) { using (profiler.Step("BPlusTree.GetNode")) { INode ret; if (_modifiedNodes.TryGetValue(nodeId, out ret)) { profiler.Incr("NodeCache Hit"); return(ret); } if (_nodeCache.TryGetValue(nodeId, out ret)) { profiler.Incr("NodeCache Hit"); return(ret); } profiler.Incr("NodeCache Miss"); using (profiler.Step("Load Node")) { var nodePage = _pageStore.Retrieve(nodeId, profiler); var header = BitConverter.ToInt32(nodePage, 0); if (header < 0) { ret = new InternalNode(nodeId, nodePage, ~header, _config); } else { ret = new LeafNode(nodeId, nodePage, header, _config); } _nodeCache.Add(ret); return(ret); } } }
public INode GetNode(ulong nodeId, BrightstarProfiler profiler) { using (profiler.Step("BPlusTree.GetNode")) { INode ret; if (_nodeCache.TryGetValue(nodeId, out ret)) { profiler.Incr("NodeCache Hit"); return(ret); } profiler.Incr("NodeCache Miss"); using (profiler.Step("Load Node")) { var nodePage = _pageStore.Retrieve(nodeId, profiler); var header = BitConverter.ToInt32(nodePage.Data, 0); if (header < 0) { ret = MakeInternalNode(nodePage, ~header); #if DEBUG_BTREE _config.BTreeDebug("{0}: Loaded INTERNAL node from page {1}. {2}", _config.DebugId, nodePage.Id, ret.ToString()); #endif } else { ret = MakeLeafNode(nodePage, header); #if DEBUG_BTREE _config.BTreeDebug("{0}: Loaded LEAF node from page {1}. {2}", _config.DebugId, nodePage.Id, ret.ToString()); #endif } _nodeCache.Add(ret); return(ret); } } }
public IPage Retrieve(ulong pageId, BrightstarProfiler profiler) { using (profiler.Step("PageStore.Retrieve")) { if (!_readonly && pageId >= _newPageOffset) { var newPageRef = _newPages[(int)(pageId - _newPageOffset)]; if (newPageRef.IsAlive) { var newPage = newPageRef.Target as IPage; if (newPage != null) { return(newPage); } } } var page = PageCache.Instance.Lookup(_path, pageId) as IPage; if (page != null) { profiler.Incr("PageCache Hit"); return(page); } if (_backgroundPageWriter != null) { // See if the page is currently queued for background write if (_backgroundPageWriter.TryGetPage(pageId, out page)) { profiler.Incr("BackgroundWriter Queue Hit"); return(page); } } using (profiler.Step("Load Page")) { profiler.Incr("PageCache Miss"); using (profiler.Step("Create FilePage")) { page = new FilePage(_stream, pageId, _pageSize); if (_backgroundPageWriter != null) { _backgroundPageWriter.ResetTimestamp(pageId); } #if DEBUG_PAGESTORE Logging.LogDebug("Load {0} {1}", pageId, BitConverter.ToInt32(page.Data, 0)); #endif } using (profiler.Step("Add FilePage To Cache")) { PageCache.Instance.InsertOrUpdate(_path, page); } return(page); } } }
public void Insert(ulong txnId, byte[] key, byte[] value, bool overwrite = false, BrightstarProfiler profiler = null) { using (profiler.Step("LeafNode.Insert")) { if (IsFull) { throw new NodeFullException(); } int insertIndex = Search(key); if (insertIndex >= 0) { if (overwrite) { EnsureWriteable(txnId); _page.SetData(value, 0, ValueOffset(insertIndex), Math.Min(value.Length, _config.ValueSize)); return; } throw new DuplicateKeyException(); } EnsureWriteable(txnId); insertIndex = ~insertIndex; RightShiftFrom(insertIndex, 1); _page.SetData(key, 0, KeyOffset(insertIndex), _config.KeySize); if (_config.ValueSize > 0 && value != null) { _page.SetData(value, 0, ValueOffset(insertIndex), Math.Min(_config.ValueSize, value.Length)); } KeyCount++; #if DEBUG_BTREE _config.BTreeDebug("LeafNode.Insert@{0}. Key={1}. Updated Node: {2}", PageId, key.Dump(), DumpKeys()); #endif } }
public void TestSimpleJoinQuery() { var sparqlQuery = @"PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/> SELECT ?review WHERE { ?review bsbm:reviewFor ?product . ?product bsbm:productFeature <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature2330> . } LIMIT 3"; // Warm-up var client = BrightstarService.GetClient("type=embedded;storesDirectory=stores"); for (int i = 0; i < 5; i++) { client.ExecuteQuery("SparqlPerformance", sparqlQuery); } // Profile var profiler = new BrightstarProfiler("SimpleJoinQuery"); for (int i = 0; i < 1000; i++) { using (profiler.Step("ExecuteQuery")) { client.ExecuteQuery("SparqlPerformance", sparqlQuery); } } Console.WriteLine(profiler.GetLogString()); }
/// <summary> /// Retrieve the resource at the specified page and segment offset /// </summary> /// <param name="pageId">The ID of the page that holds the resource to be retrieved</param> /// <param name="segment">The index of the segment within the page that holds the start of the resource</param> /// <param name="profiler"></param> /// <returns>The resource</returns> public string GetResource(ulong pageId, byte segment, BrightstarProfiler profiler) { using (profiler.Step("ResourceTable.GetResource")) { var currentPage = _pageStore.Retrieve(pageId, profiler); int resourceLength = BitConverter.ToInt32(currentPage, segment*_segmentSize); int totalLength = resourceLength + 4; int segmentsToLoad = totalLength/_segmentSize; if (totalLength%_segmentSize > 0) segmentsToLoad++; var buffer = new byte[segmentsToLoad*_segmentSize]; byte segmentIndex = segment; for (int i = 0; i < segmentsToLoad; i++) { if (segmentIndex == _pointerSegment) { ulong nextPageId = BitConverter.ToUInt64(currentPage, _pageStore.PageSize - 8); currentPage = _pageStore.Retrieve(nextPageId, profiler); segmentIndex = 0; } Array.Copy(currentPage, segmentIndex*_segmentSize, buffer, i*_segmentSize, _segmentSize); segmentIndex++; } return Encoding.UTF8.GetString(buffer, 4, resourceLength); } }
public void Commit(ulong commitId, BrightstarProfiler profiler) { using (profiler.Step("PageStore.Commit")) { var pagesToWrite = _newPages.Where(rf => rf.IsAlive).Select(rf => rf.Target as IPage).Where(pg => pg != null); if (_backgroundPageWriter != null) { foreach (var page in pagesToWrite) { _backgroundPageWriter.QueueWrite(page, commitId); } _backgroundPageWriter.Flush(); RestartBackgroundWriter(); } else { using (var outputStream = _peristenceManager.GetOutputStream(_path, FileMode.Open)) { foreach (var page in pagesToWrite) { page.Write(outputStream, commitId); } } } _newPages.Clear(); _newPageOffset = _nextPageId; PageCache.Instance.Clear(_path); } }
public INode GetNode(ulong nodeId, BrightstarProfiler profiler) { using (profiler.Step("BPlusTree.GetNode")) { INode ret; if (_modifiedNodes.TryGetValue(nodeId, out ret)) { profiler.Incr("NodeCache Hit"); return ret; } if (_nodeCache.TryGetValue(nodeId, out ret)) { profiler.Incr("NodeCache Hit"); return ret; } profiler.Incr("NodeCache Miss"); using (profiler.Step("Load Node")) { var nodePage = _pageStore.Retrieve(nodeId, profiler); var header = BitConverter.ToInt32(nodePage, 0); if (header < 0) { ret = new InternalNode(nodeId, nodePage, ~header, _config); } else { ret = new LeafNode(nodeId, nodePage, header, _config); } _nodeCache.Add(ret); return ret; } } }
/// <summary> /// Commits all changed and new pages to the page store /// </summary> /// <param name="commitId">The transaction identifier for the commit</param> /// <param name="profiler"></param> public void Commit(ulong commitId, BrightstarProfiler profiler) { EnsureOutputStream(); using (profiler.Step("PageStore.Commit")) { try { foreach (var entry in _modifiedPages.OrderBy(e => e.Key)) { // TODO: Ensure we are writing the correct commit entry.Value.Item1.Write(_outputStream, commitId); lock (_pageCacheLock) { PageCache.Instance.InsertOrUpdate(_filePath, entry.Value.Item1); } } _modifiedPages.Clear(); } catch (Exception) { _modifiedPages.Clear(); throw; } CurrentTransactionId = commitId; _outputStream.Flush(); _outputStream.Close(); _outputStream = null; } }
/// <summary> /// Performs the actual load of prefixes from a page. /// </summary> /// <param name="page"></param> /// <param name="profiler"></param> /// <remarks>Calls to this method should be made inside a critical section of code protected with a mutex or reader/writer lock</remarks> private void InterlockedLoad(IPage page, BrightstarProfiler profiler) { using (profiler.Step("PrefixManager.InterlockedLoad")) { int offset = 0; while (offset < _pageStore.PageSize) { ushort prefixLength = BitConverter.ToUInt16(page.Data, offset); offset += 2; if (prefixLength == ushort.MaxValue) { ulong nextPageId = BitConverter.ToUInt64(page.Data, offset); if (nextPageId == 0) { // End of data return; } page = _pageStore.Retrieve(nextPageId, profiler); offset = 0; } else { var prefix = Encoding.UTF8.GetString(page.Data, offset, prefixLength); offset += prefixLength; var uriLen = BitConverter.ToUInt16(page.Data, offset); offset += 2; var uri = Encoding.UTF8.GetString(page.Data, offset, uriLen); offset += uriLen; _prefixMappings[uri] = prefix; _shortValueMappings[prefix] = uri; } } } }
public void Insert(ulong transactionId, string resource, out ulong pageId, out byte segmentId, BrightstarProfiler profiler) { using (profiler.Step("ResourceTable.Insert")) { var byteCount = Encoding.UTF8.GetByteCount(resource); var resourceBytes = new byte[byteCount + 4]; BitConverter.GetBytes(byteCount).CopyTo(resourceBytes, 0); Encoding.UTF8.GetBytes(resource, 0, resource.Length, resourceBytes, 4); lock (_writeLock) { if (_nextSegment == _pointerSegment) { StartNewPage(transactionId, profiler); } pageId = _currentPage; segmentId = _nextSegment; for (int i = 0; i < (byteCount + 4); i += _segmentSize) { _pageStore.Write(transactionId, _currentPage, resourceBytes, i, _nextSegment * _segmentSize, _segmentSize < (byteCount + 4 - i) ? _segmentSize : (byteCount + 4 - i), profiler); _nextSegment++; if (_nextSegment == _pointerSegment) { StartNewPage(transactionId, profiler); } } } } }
public void Delete(ulong txnId, byte[] key, BrightstarProfiler profiler) { using (profiler.Step("BPlusTree.Delete")) { var root = GetNode(_rootId, profiler); if (root is ILeafNode) { (root as ILeafNode).Delete(txnId, key); MarkDirty(txnId, root, profiler); // Update root page pointer - see note in Insert() method above _rootId = root.PageId; } else { bool underAllocation; Delete(txnId, root as IInternalNode, key, out underAllocation, profiler); if (root.KeyCount == 0) { // Now has only a single child leaf node, which should become the new tree root root = GetNode((root as IInternalNode).GetChildPointer(0), profiler); _rootId = root.PageId; } else { // Update root page pointer - see note in Insert() method above _rootId = root.PageId; } } } }
/// <summary> /// Retrieves the resource with the specified resource ID /// </summary> /// <param name="resourceId">The resource ID to look for</param> /// <param name="addToCache">Boolean flag indicating if the returned resource structure should be cached for faster future lookups</param> /// <param name="profiler">OPTIONAL: Profiler to use to profile this call</param> /// <returns>The corresponding resource or null if no match is found</returns> public IResource GetResource(ulong resourceId, bool addToCache, BrightstarProfiler profiler = null) { using (profiler.Step("ResourceIndex.GetResource")) { IResource resource; if (_resourceCache.TryGetValue(resourceId, out resource)) { return(resource); } var buff = new byte[64]; if (Search(resourceId, buff, profiler)) { resource = _resourceStore.FromBTreeValue(buff); _resourceCache.Add(resourceId, resource); return(resource); } #if DEBUG if (resourceId > 0) { // Repeat the search for debug purposes Search(resourceId, buff, profiler); } #endif return(null); } }
public IEnumerable <KeyValuePair <byte[], byte []> > Scan(BrightstarProfiler profiler) { using (profiler.Step("Scan Entire BTree")) { return(Scan(_root, profiler)); } }
public void Write(ulong commitId, ulong pageId, byte[] data, int srcOffset = 0, int pageOffset = 0, int len = -1, BrightstarProfiler profiler = null) { if (pageId < _newPageOffset) { throw new InvalidOperationException("Attempt to write to a fixed page"); } var pageIx = (int)(pageId - _newPageOffset); if (pageIx >= _newPages.Count) { throw new InvalidOperationException("Attempt to write to an unreserved page"); } using (profiler.Step("Write Page")) { var page = Retrieve(pageId, profiler); page.SetData(data, srcOffset, pageOffset, len); if (_backgroundPageWriter != null) { #if DEBUG_PAGESTORE Logging.LogDebug("Mark {0}", page.Id); #endif _backgroundPageWriter.QueueWrite(page, commitId); } } }
/// <summary> /// Retrieve the resource at the specified page and segment offset /// </summary> /// <param name="pageId">The ID of the page that holds the resource to be retrieved</param> /// <param name="segment">The index of the segment within the page that holds the start of the resource</param> /// <param name="profiler"></param> /// <returns>The resource</returns> public string GetResource(ulong pageId, byte segment, BrightstarProfiler profiler) { using (profiler.Step("ResourceTable.GetResource")) { var currentPage = _pageStore.Retrieve(pageId, profiler); int resourceLength = BitConverter.ToInt32(currentPage, segment * _segmentSize); int totalLength = resourceLength + 4; int segmentsToLoad = totalLength / _segmentSize; if (totalLength % _segmentSize > 0) { segmentsToLoad++; } var buffer = new byte[segmentsToLoad * _segmentSize]; byte segmentIndex = segment; for (int i = 0; i < segmentsToLoad; i++) { if (segmentIndex == _pointerSegment) { ulong nextPageId = BitConverter.ToUInt64(currentPage, _pageStore.PageSize - 8); currentPage = _pageStore.Retrieve(nextPageId, profiler); segmentIndex = 0; } Array.Copy(currentPage, segmentIndex * _segmentSize, buffer, i * _segmentSize, _segmentSize); segmentIndex++; } return(Encoding.UTF8.GetString(buffer, 4, resourceLength)); } }
private PredicateRelatedResourceIndex AssertPredicateIndex(ulong transactionId, ulong predicateId, BrightstarProfiler profiler) { using (profiler.Step("AssertPredicateIndex")) { PredicateRelatedResourceIndex predicateIndex = GetPredicateIndex(predicateId, profiler); if (predicateIndex == null) { using (profiler.Step("New Predicate Index")) { predicateIndex = new PredicateRelatedResourceIndex(transactionId, PageStore); _predicateIndexes[predicateId] = predicateIndex; } } return(predicateIndex); } }
public void TestSimpleJoinQuery() { var sparqlQuery = @"PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/> SELECT ?review WHERE { ?review bsbm:reviewFor ?product . ?product bsbm:productFeature <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature2330> . } LIMIT 3"; // Warm-up var client = BrightstarService.GetClient("type=embedded;storesDirectory=stores"); for (int i = 0; i < 5; i++) { client.ExecuteQuery("SparqlPerformance", sparqlQuery); } // Profile var profiler = new BrightstarProfiler("SimpleJoinQuery"); for (int i = 0; i < 100; i++) { using (profiler.Step("ExecuteQuery")) { client.ExecuteQuery("SparqlPerformance", sparqlQuery); } } Console.WriteLine(profiler.GetLogString()); }
/// <summary> /// Commits all changed and new pages to the page store /// </summary> /// <param name="commitId">The transaction identifier for the commit</param> /// <param name="profiler"></param> public void Commit(ulong commitId, BrightstarProfiler profiler) { using (profiler.Step("PageStore.Commit")) { using (var outputStream = _persistenceManager.GetOutputStream(_filePath, FileMode.Open)) { try { foreach (var entry in _modifiedPages.OrderBy(e => e.Key)) { entry.Value.Write(outputStream, commitId); lock (_pageCacheLock) { PageCache.Instance.InsertOrUpdate(_filePath, entry.Value); } } _modifiedPages.Clear(); } catch (Exception) { _modifiedPages.Clear(); throw; } } CurrentTransactionId = commitId; } }
public void Triple(string subject, bool subjectIsBNode, string predicate, bool predicateIsBNode, string obj, bool objIsBNode, bool isLiteral, string dataType, string langCode, string graphUri) { #if DEBUG_INSERTS Logging.LogDebug("Triple {0} {1} {2} {3} {4} {5} {6} {7} {8} {9}", subject, subjectIsBNode, predicate, predicateIsBNode, obj, objIsBNode, isLiteral, dataType, langCode, graphUri); #endif using (_profiler.Step("Assert BNodes")) { // Convert BNode identifiers to internal BNode identifiers) if (subjectIsBNode) { subject = AssertBNode(subject); } if (predicateIsBNode) { predicate = AssertBNode(predicate); } if (objIsBNode) { obj = AssertBNode(obj); } } #if DEBUG_INSERTS Logging.LogDebug("InsertTriple {0} {1} {2} {3} {4} {5} {6}", subject, predicate, obj, isLiteral, dataType, langCode, graphUri); #endif _store.InsertTriple(subject, predicate, obj, isLiteral, dataType, langCode, graphUri, _profiler); _total++; if (_total % _batchSize == 0) { if (_commitEachBatch) { Logging.LogInfo("Committing..."); _store.Commit(_jobId, _profiler); } #if MONITOR_MEMORY_LOAD else { if (Server.MemoryUtils.GetMemoryLoad() > HighMemoryLoad) { Logging.LogInfo("Flushing..."); _store.FlushChanges(_profiler); // Clear out any interned URIs to free up more memory VDS.RDF.UriFactory.Clear(); Logging.LogDebug("Initiating garbage collection..."); GC.Collect(); } } #endif Logging.LogInfo("Complete @ " + _total); } }
public virtual ulong Save(ulong transactionId, BrightstarProfiler profiler) { using (profiler.Step("BPlusTree.Save")) { _isDirty = false; return(RootId); } }
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler) { using (profiler.Step("RelatedResourceIndex.Write")) { var indexBuilder = new BPlusTreeBuilder(pageStore, Configuration); return(indexBuilder.Build(transactionId, WritePredicateIndexes(pageStore, transactionId, profiler), profiler)); } }
/// <summary> /// Finds or creates a new ID for the graph with the specified graph URI /// </summary> /// <param name="graphUri">The graph URI to lookup</param> /// <param name="profiler"></param> /// <returns>The ID assigned to the graph</returns> public int AssertGraphId(string graphUri, BrightstarProfiler profiler = null) { if (String.IsNullOrEmpty(graphUri)) { throw new ArgumentException("Graph URI must not be null or an empty string", "graphUri"); } if (graphUri.Length > short.MaxValue) { throw new ArgumentException( String.Format("Graph URI string exceeds maximum allowed length of {0} bytes", short.MaxValue), "graphUri"); } #if WINDOWS_PHONE || PORTABLE lock (_lock) { int entryId; if (_graphUriIndex.TryGetValue(graphUri, out entryId) && !_allEntries[entryId].IsDeleted) { return(entryId); } var newId = _allEntries.Count; var entry = new GraphIndexEntry(newId, graphUri, false); _allEntries.Add(entry); _graphUriIndex.Add(graphUri, newId); return(newId); } #else using (profiler.Step("Assert Graph Id")) { _lock.EnterUpgradeableReadLock(); try { int entryId; if (_graphUriIndex.TryGetValue(graphUri, out entryId) && !_allEntries[entryId].IsDeleted) { return(entryId); } _lock.EnterWriteLock(); try { var newId = _allEntries.Count; var entry = new GraphIndexEntry(newId, graphUri, false); _allEntries.Add(entry); _graphUriIndex.Add(graphUri, newId); return(newId); } finally { _lock.ExitWriteLock(); } } finally { _lock.ExitUpgradeableReadLock(); } } #endif }
/// <summary> /// Adds a related resource index entry /// </summary> /// <param name="txnId"> </param> /// <param name="resourceId">The resource ID of the "start" resource</param> /// <param name="predicateId">The resource ID of the predicate that relates the resources</param> /// <param name="relatedResourceId">The resource ID of the "related" resource</param> /// <param name="graphId">The resource ID of the graph containing the relationship</param> /// <param name="profiler"></param> public void AddRelatedResource(ulong txnId, ulong resourceId, ulong predicateId, ulong relatedResourceId, int graphId, BrightstarProfiler profiler = null) { using (profiler.Step("Add Related Resource")) { var predicateIndex = AssertPredicateIndex(txnId, predicateId, profiler); var key = MakePredicateIndexKey(resourceId, graphId, relatedResourceId); try { using (profiler.Step("Insert Into Predicate Index")) { predicateIndex.Insert(txnId, key, null, profiler: profiler); } } catch (DuplicateKeyException) { // Ignore duplicate key exceptions } } }
/// <summary> /// Adds a related resource index entry /// </summary> /// <param name="txnId"> </param> /// <param name="resourceId">The resource ID of the "start" resource</param> /// <param name="predicateId">The resource ID of the predicate that relates the resources</param> /// <param name="relatedResourceId">The resource ID of the "related" resource</param> /// <param name="graphId">The resource ID of the graph containing the relationship</param> /// <param name="profiler"></param> public void AddRelatedResource(ulong txnId, ulong resourceId, ulong predicateId, ulong relatedResourceId, int graphId, BrightstarProfiler profiler = null) { using (profiler.Step("Add Related Resource")) { var predicateIndex = AssertPredicateIndex(txnId, predicateId, profiler); var key = MakePredicateIndexKey(resourceId, graphId, relatedResourceId); try { using (profiler.Step("Insert Into Predicate Index")) { predicateIndex.Insert(txnId, key, null, profiler: profiler); } } catch (DuplicateKeyException) { // Ignore duplicate key exceptions } } }
public ConcurrentGraphIndex(IPageStore pageStore, ulong rootPage, BrightstarProfiler profiler) { using (profiler.Step("Load ConcurrentGraphIndex")) { _pageStore = pageStore; _graphUriIndex = new Dictionary <string, int>(); _allEntries = new List <GraphIndexEntry>(); Read(rootPage, profiler); } }
public static IPageStore CreateEmptyPageStore(string fileName, PersistenceType persistenceType = PersistenceType.AppendOnly, BrightstarProfiler profiler = null) { using (profiler.Step("Create Empty Page Store")) { using (profiler.Step("Delete Existing")) { if (PersistenceManager.FileExists(fileName)) PersistenceManager.DeleteFile(fileName); //if (File.Exists(fileName)) File.Delete(fileName); } using (profiler.Step("Create New Page Store")) { if (persistenceType == PersistenceType.AppendOnly) { return new AppendOnlyFilePageStore(PersistenceManager, fileName, 4096, false, false); } return new BinaryFilePageStore(PersistenceManager, fileName, 4096, false, 1); } } }
public ConcurrentGraphIndex(IPageStore pageStore, ulong rootPage, BrightstarProfiler profiler) { using (profiler.Step("Load ConcurrentGraphIndex")) { _pageStore = pageStore; _graphUriIndex = new Dictionary<string, int>(); _allEntries = new List<GraphIndexEntry>(); Read(rootPage, profiler); } }
/// <summary> /// Writes data to the specified page /// </summary> /// <param name="commitId">The transaction id for the update</param> /// <param name="pageId">The ID of the page to write to</param> /// <param name="buff">The data to be written</param> /// <param name="srcOffset">The offset into <paramref name="buff"/> from which to start copying bytes. Defaults to 0</param> /// <param name="pageOffset">The offset into the page data buffer to start writing to. Defaults to 0</param> /// <param name="len">The number of bytes to write. Defaults to all bytes in <paramref name="buff"/> from the specified <paramref name="srcOffset"/></param> /// <param name="profiler"></param> public void Write(ulong commitId, ulong pageId, byte[] buff, int srcOffset = 0, int pageOffset = 0, int len = -1, BrightstarProfiler profiler = null) { using (profiler.Step("Write Page")) { var page = GetPage(pageId, profiler); var writeBuff = page.GetWriteBuffer(commitId); Array.Copy(buff, srcOffset, writeBuff, pageOffset, len < 0 ? buff.Length : len); _modifiedPages[pageId] = new Tuple <BinaryFilePage, ulong>(page, commitId); } }
public Store(string storeLocation, IPageStore dataPageStore, IResourceTable resourceTable, ulong storePageId, BrightstarProfiler profiler) { using (profiler.Step("Load Store")) { DirectoryPath = storeLocation; _pageStore = dataPageStore; _resourceTable = resourceTable; var storePage = _pageStore.Retrieve(storePageId, profiler); Load(storePage, profiler); } }
public bool ContainsRelatedResource(ulong resourceId, ulong predicateId, ulong relatedResourceId, int graphId, BrightstarProfiler profiler) { using (profiler.Step("ContainsRelatedResource")) { byte[] valueBuff = null; PredicateRelatedResourceIndex predicateIndex = GetPredicateIndex(predicateId, profiler); return(predicateIndex != null && predicateIndex.Search(MakePredicateIndexKey(resourceId, graphId, relatedResourceId), valueBuff, profiler)); } }
/// <summary> /// Enumerates the key-value pairs stored in the BTree starting with <paramref name="fromKey"/> /// up to <paramref name="toKey"/> (inclusive) /// </summary> /// <param name="fromKey">The lowest key to return in the enumeration</param> /// <param name="toKey">The highest key to return in the enumeration</param> /// <param name="profiler"></param> /// <returns>An enumeration of key-value pairs from the BTree</returns> public IEnumerable <KeyValuePair <byte[], byte[]> > Scan(byte[] fromKey, byte[] toKey, BrightstarProfiler profiler) { using (profiler.Step("Scan BTree Range")) { if (fromKey.Compare(toKey) > 1) { throw new ArgumentException("Scan can only be performed in increasing order."); } return(Scan(_root, fromKey, toKey, profiler)); } }
public static IPageStore OpenPageStore(string fileName, bool readOnly, PersistenceType persistenceType = PersistenceType.AppendOnly, ulong txnId = 1UL, BrightstarProfiler profiler = null) { using (profiler.Step("Open Page Store")) { if (persistenceType == PersistenceType.AppendOnly) { return new AppendOnlyFilePageStore(PersistenceManager, fileName, 4096, readOnly, false); } return new BinaryFilePageStore(PersistenceManager, fileName, 4096, readOnly, txnId); } }
public Store(string storeLocation, IPageStore dataPageStore, IResourceTable resourceTable, ulong storePageId, BrightstarProfiler profiler) { using (profiler.Step("Load Store")) { DirectoryPath = storeLocation; _pageStore = dataPageStore; _resourceTable = resourceTable; var storePage = _pageStore.Retrieve(storePageId, profiler); Load(storePage, profiler); } }
/// <summary> /// Enumerates the resources related to a start resource by a specific predicate /// </summary> /// <param name="resourceId">The resource ID of the start resource</param> /// <param name="predicateId">The resource ID of the predicate that relates the resources. If set to Constants.NullUlong, returns relationships for all predicates</param> /// <param name="graphId">The resource ID of the graph containing the relationships. If set to Constants.NullUlong, returns relationships from all graphs</param> /// <param name="profiler"></param> /// <returns></returns> public IEnumerable<IRelatedResource> EnumerateRelatedResources(ulong resourceId, ulong predicateId = StoreConstants.NullUlong, int graphId = -1, BrightstarProfiler profiler = null) { using (profiler.Step("EnumerateRelatedResources")) { if (predicateId != StoreConstants.NullUlong) { byte[] minKey = MakePredicateIndexKey(resourceId, graphId < 0 ? 0 : graphId, UInt64.MinValue); byte[] maxKey = MakePredicateIndexKey(resourceId, graphId < 0 ? Int32.MaxValue : graphId, UInt64.MaxValue); var predicateIndex = GetPredicateIndex(predicateId, profiler); if (predicateIndex != null) { foreach (var r in predicateIndex.Scan(minKey, maxKey, profiler).Select( x => new RelatedResource(predicateId, GetGraphIdFromKey(x.Key), GetRelatedResourceIdFromKey(x.Key)))) { yield return r; } } } else { foreach (var entry in Scan(0ul, UInt64.MaxValue, profiler)) { // Load the predicate index into the cache GetPredicateIndex(entry.Key,BitConverter.ToUInt64(entry.Value, 0), profiler); // Then a recursive call to enumerate the index tree foreach (var r in EnumerateRelatedResources(resourceId, entry.Key, graphId, profiler)) { yield return r; } } } } }
/// <summary> /// Ensures that the specified resource is in the resource index and returns its resource ID /// </summary> /// <param name="txnId">The ID of the current update transaction</param> /// <param name="resourceValue">The resource string value</param> /// <param name="isLiteral">Boolean flag indicating if the resource is a literal (true) or a URI (false)</param> /// <param name="dataType">The resource data-type URI</param> /// <param name="langCode">The resource language string</param> /// <param name="addToCache">Boolean flag indicating if newly indexed resources should be added to the local cache</param> /// <param name="profiler"></param> /// <returns>The resource ID for the resource</returns> public ulong AssertResourceInIndex(ulong txnId, string resourceValue, bool isLiteral = false, string dataType = null, string langCode = null, bool addToCache = true, BrightstarProfiler profiler = null) { using (profiler.Step("AssertResourceInIndex")) { // Normalize language code to null if it is an empty string if (String.IsNullOrEmpty(langCode)) { langCode = null; } // Retrieve the resource ID for the datatype URI (if any) var dataTypeId = String.IsNullOrEmpty(dataType) ? StoreConstants.NullUlong : AssertResourceInIndex(txnId, dataType, profiler:profiler); var hashString = isLiteral ? MakeHashString(resourceValue, dataType, langCode) : resourceValue; ulong resourceId; if (_resourceIdCache.TryGetValue(hashString, out resourceId)) { return resourceId; } // Get a ulong resource ID for the language code string var langCodeId = String.IsNullOrEmpty(langCode) ? StoreConstants.NullUlong : AssertResourceInIndex(txnId, langCode, true, profiler:profiler); resourceId = AssertResourceInBTree(txnId, resourceValue, isLiteral, dataTypeId, langCodeId, StringExtensions.GetBrightstarHashCode(hashString), profiler); if (addToCache) { _resourceIdCache.Add(hashString, resourceId); } return resourceId; } }
/* * Data format for the store page: * * 00-03: Store format version (int) * 04-11: Transaction id (ulong) * 12-19: Graph Index start page id (ulong) * 20-27: Prefix Manager start page id (ulong) * 28-35: Resource Index start page id (ulong) * 36-43: Subject Related Resource Index start page id (ulong) * 44-51: Object Related Resource Index start page id (ulong) * 44-107: Reserved (all zeros for now) * 108-127: SHA1 hash of bytes 00-108 * 128-255: Repeat of the above structure */ private bool Load(IPage storePage, BrightstarProfiler profiler) { using (profiler.Step("Store.Load")) { // Validate the hash for the index bloc using (var sha1 = new SHA1Managed()) { var recordedHash = new byte[20]; Array.Copy(storePage.Data, 108, recordedHash, 0, 20); var calculatedHash = sha1.ComputeHash(storePage.Data, 0, 108); if (recordedHash.Compare(calculatedHash) != 0) { return false; } } // Load indexes from the pointers int storeVersion = BitConverter.ToInt32(storePage.Data, 0); if (storeVersion == 1) { _currentTxnId = BitConverter.ToUInt64(storePage.Data, 4); var graphIndexId = BitConverter.ToUInt64(storePage.Data, 12); _graphIndex = new ConcurrentGraphIndex(_pageStore, graphIndexId, profiler); var prefixManagerId = BitConverter.ToUInt64(storePage.Data, 20); _prefixManager = new PrefixManager(_pageStore, prefixManagerId, profiler); var resourceIndexId = BitConverter.ToUInt64(storePage.Data, 28); _resourceIndex = new ResourceIndex.ResourceIndex(_pageStore, _resourceTable, resourceIndexId); var relatedResourceIndexId = BitConverter.ToUInt64(storePage.Data, 36); _subjectRelatedResourceIndex = new RelatedResourceIndex.RelatedResourceIndex(_pageStore, relatedResourceIndexId, profiler); var objectRelatedResourceIndexId = BitConverter.ToUInt64(storePage.Data, 44); _objectRelatedResourceIndex = new RelatedResourceIndex.RelatedResourceIndex(_pageStore, objectRelatedResourceIndexId, profiler); } return true; } }
private ulong Insert(ulong txnId, INode node, byte[] key, byte[] value, out bool split, out INode rightNode, out byte[] splitKey, bool overwrite, BrightstarProfiler profiler) { if (node is LeafNode) { var leaf = node as LeafNode; if (leaf.IsFull) { var newNode = leaf.Split(_pageStore.Create(), out splitKey); _modifiedNodes[newNode.PageId] = newNode; if (key.Compare(splitKey) < 0) { leaf.Insert(key, value, overwrite, profiler); } else { newNode.Insert(key, value, overwrite, profiler); } MarkDirty(txnId, leaf, profiler); split = true; rightNode = newNode; } else { leaf.Insert(key, value, overwrite, profiler); MarkDirty(txnId, leaf, profiler); split = false; rightNode = null; splitKey = null; } return leaf.PageId; } else { var internalNode = node as InternalNode; var childNodeId = internalNode.GetChildNodeId(key); var childNode = GetNode(childNodeId, profiler); bool childSplit; INode rightChild; byte[] childSplitKey; var newChildNodeId = Insert(txnId, childNode, key, value, out childSplit, out rightChild, out childSplitKey, overwrite, profiler); if (childSplit) { if (internalNode.IsFull) { using (profiler.Step("Split Internal Node")) { // Need to split this node to insert the new child node rightNode = internalNode.Split(_pageStore.Create(), out splitKey); _modifiedNodes[rightNode.PageId] = rightNode; split = true; if (childSplitKey.Compare(splitKey) < 0) { internalNode.Insert(childSplitKey, rightChild.PageId); } else { (rightNode as InternalNode).Insert(childSplitKey, rightChild.PageId); } // update child pointers if required (need to check both internalNode and rightNode as we don't know which side the modified child node ended up on) if (newChildNodeId != childNodeId) { internalNode.UpdateChildPointer(childNodeId, newChildNodeId); (rightNode as InternalNode).UpdateChildPointer(childNodeId, newChildNodeId); } } } else { using (profiler.Step("Insert into internal node")) { split = false; rightNode = null; splitKey = null; internalNode.Insert(childSplitKey, rightChild.PageId); } } using (profiler.Step("Update Child Pointer")) { if (newChildNodeId != childNodeId) { internalNode.UpdateChildPointer(childNodeId, newChildNodeId); } MarkDirty(txnId, internalNode, profiler); } return internalNode.PageId; } else { using (profiler.Step("Update Child Pointer")) { if (newChildNodeId != childNodeId) { internalNode.UpdateChildPointer(childNodeId, newChildNodeId); MarkDirty(txnId, internalNode, profiler); } split = false; rightNode = null; splitKey = null; return internalNode.PageId; } } } }
public void Commit(Guid jobId, BrightstarProfiler profiler = null) { using (profiler.Step("Store Commit")) { ulong storePageId = Save(profiler); var storeManager = StoreManagerFactory.GetStoreManager(); var mf = storeManager.GetMasterFile(DirectoryPath); mf.AppendCommitPoint(new CommitPoint(storePageId,_currentTxnId + 1,DateTime.UtcNow, jobId)); _currentTxnId++; } }
/// <summary> /// Insert a triple into the store /// </summary> /// <param name="subject"></param> /// <param name="predicate"></param> /// <param name="objValue"></param> /// <param name="isObjectLiteral"></param> /// <param name="dataType"></param> /// <param name="langCode"></param> /// <param name="graphUri"></param> /// <param name="profiler"></param> public void InsertTriple(string subject, string predicate, string objValue, bool isObjectLiteral, string dataType, string langCode, string graphUri, BrightstarProfiler profiler = null) { using (profiler.Step("InsertTriple")) { using (profiler.Step("Normalization")) { // Normalize subject, predicate, objValue (if it is not a literal), dataType (if it is not null) // and graphUri (if it is not null) try { subject = new Uri(subject, UriKind.Absolute).ToString(); } catch (FormatException) { throw new InvalidTripleException( String.Format("The subject '{0}' could not be parsed as a valid URI.", subject)); } try { predicate = new Uri(predicate, UriKind.Absolute).ToString(); } catch (FormatException) { throw new InvalidTripleException( String.Format("The predicate'{0}' could not be parsed as a valid URI.", predicate)); } if (!isObjectLiteral) { try { objValue = new Uri(objValue, UriKind.Absolute).ToString(); } catch (FormatException) { throw new InvalidTripleException( String.Format("The object '{0}' could not be parsed as a valid URI.", objValue)); } } if (isObjectLiteral && !String.IsNullOrEmpty(dataType)) { try { dataType = new Uri(dataType, UriKind.Absolute).ToString(); } catch (FormatException) { throw new InvalidTripleException( String.Format("The dataType '{0}' could not be parsed as a valid URI.", dataType)); } } if (!String.IsNullOrEmpty(graphUri)) { try { graphUri = new Uri(graphUri, UriKind.Absolute).ToString(); } catch (FormatException) { throw new InvalidTripleException( String.Format("The graphUri '{0}' could not be parsed as a valid URI.", graphUri)); } } if (isObjectLiteral && dataType == null) { dataType = RdfDatatypes.PlainLiteral; } // Normalize language code to lower-case (per http://www.w3.org/TR/rdf-concepts/#section-Graph-Literal) if (langCode != null) { langCode = langCode.ToLowerInvariant(); } } using (profiler.Step("Make Prefixed Uris")) { subject = _prefixManager.MakePrefixedUri(subject); predicate = _prefixManager.MakePrefixedUri(predicate); if (!isObjectLiteral) { objValue = _prefixManager.MakePrefixedUri(objValue); } } var txnId = _currentTxnId + 1; ulong sid = _resourceIndex.AssertResourceInIndex(txnId, subject, profiler:profiler); ulong pid = _resourceIndex.AssertResourceInIndex(txnId, predicate, profiler:profiler); ulong oid = _resourceIndex.AssertResourceInIndex(txnId, objValue, isObjectLiteral, dataType, langCode, !isObjectLiteral, profiler); int gid = _graphIndex.AssertGraphId(graphUri, profiler); _subjectRelatedResourceIndex.AddRelatedResource(txnId, sid, pid, oid, gid, profiler); _objectRelatedResourceIndex.AddRelatedResource(txnId, oid, pid, sid, gid, profiler); } }
public void Commit(ulong transactionId, BrightstarProfiler profiler) { using (profiler.Step("Commit ResourceTable")) { if (_currentPage > 0) { _pageStore.Commit(transactionId, profiler); _nextSegment = _pointerSegment; _currentPage = 0; } } }
/// <summary> /// Enumerates the key-value pairs stored in the BTree starting with <paramref name="fromKey"/> /// up to <paramref name="toKey"/> (inclusive) /// </summary> /// <param name="fromKey">The lowest key to return in the enumeration</param> /// <param name="toKey">The highest key to return in the enumeration</param> /// <param name="profiler"></param> /// <returns>An enumeration of key-value pairs from the BTree</returns> public IEnumerable<KeyValuePair<byte[], byte[]>> Scan(byte[] fromKey, byte[] toKey, BrightstarProfiler profiler ) { using (profiler.Step("Scan BTree Range")) { if (fromKey.Compare(toKey) > 1) { throw new ArgumentException("Scan can only be performed in increasing order."); } return Scan(_root, fromKey, toKey, profiler); } }
public void Insert(ulong txnId, byte[] key, byte[] value, bool overwrite = false, BrightstarProfiler profiler = null) { using (profiler.Step("BPlusTree.Insert")) { bool splitRoot; INode rightNode; byte[] rootSplitKey; Insert(txnId, _root, key, value, out splitRoot, out rightNode, out rootSplitKey, overwrite, profiler); if (splitRoot) { var newRoot = new InternalNode(_pageStore.Create(), rootSplitKey, _root.PageId, rightNode.PageId, _config); MarkDirty(txnId, _root, profiler); _modifiedNodes[newRoot.PageId] = newRoot; _root = newRoot; } } }
public void Insert(ulong transactionId, string resource, out ulong pageId, out byte segmentId, BrightstarProfiler profiler) { using (profiler.Step("ResourceTable.Insert")) { var byteCount = Encoding.UTF8.GetByteCount(resource); var resourceBytes = new byte[byteCount + 4]; BitConverter.GetBytes(byteCount).CopyTo(resourceBytes, 0); Encoding.UTF8.GetBytes(resource, 0, resource.Length, resourceBytes, 4); lock (_writeLock) { if (_nextSegment == _pointerSegment) { StartNewPage(transactionId, profiler); } pageId = _currentPage; segmentId = _nextSegment; for (int i = 0; i < (byteCount + 4); i += _segmentSize) { _pageStore.Write(transactionId, _currentPage, resourceBytes, i, _nextSegment*_segmentSize, _segmentSize < (byteCount + 4 - i) ? _segmentSize : (byteCount + 4 - i), profiler); _nextSegment++; if (_nextSegment == _pointerSegment) { StartNewPage(transactionId, profiler); } } } } }
public void Commit(ulong commitId, BrightstarProfiler profiler) { using (profiler.Step("PageStore.Commit")) { var pagesToWrite = _newPages.Where(rf => rf.IsAlive).Select(rf => rf.Target as IPage).Where(pg => pg != null); if (_backgroundPageWriter != null) { foreach (var page in pagesToWrite) { _backgroundPageWriter.QueueWrite(page, commitId); } _backgroundPageWriter.Flush(); RestartBackgroundWriter(); } else { using (var outputStream = _peristenceManager.GetOutputStream(_path, FileMode.Open)) { foreach (var page in pagesToWrite) { page.Write(outputStream, commitId); } } } _newPages.Clear(); _newPageOffset = _nextPageId; PageCache.Instance.Clear(_path); } }
public virtual ulong Save(ulong transactionId, BrightstarProfiler profiler) { using (profiler.Step("BPlusTree.Save")) { foreach (var n in _modifiedNodes.Values) { _pageStore.Write(transactionId, n.PageId, n.GetData(), profiler: profiler); n.IsDirty = false; //_nodeCache.Add(n.PageId, new WeakReference(n)); _nodeCache.Add(n); } _modifiedNodes.Clear(); return RootId; } }
/// <summary> /// Performs the actual load of prefixes from a page. /// </summary> /// <param name="page"></param> /// <param name="profiler"></param> /// <remarks>Calls to this method should be made inside a critical section of code protected with a mutex or reader/writer lock</remarks> private void InterlockedLoad(byte[] page, BrightstarProfiler profiler) { using (profiler.Step("PrefixManager.InterlockedLoad")) { int offset = 0; while (offset < _pageStore.PageSize) { ushort prefixLength = BitConverter.ToUInt16(page, offset); offset += 2; if (prefixLength == ushort.MaxValue) { ulong nextPageId = BitConverter.ToUInt64(page, offset); if (nextPageId == 0) { // End of data return; } page = _pageStore.Retrieve(nextPageId, profiler); offset = 0; } else { var prefix = Encoding.UTF8.GetString(page, offset, prefixLength); offset += prefixLength; var uriLen = BitConverter.ToUInt16(page, offset); offset += 2; var uri = Encoding.UTF8.GetString(page, offset, uriLen); offset += uriLen; _prefixMappings[uri] = prefix; _shortValueMappings[prefix] = uri; } } } }
public bool Search(byte[] key, byte[] valueBuff, BrightstarProfiler profiler) { using (profiler.Step("BPlusTree.Search")) { INode u = _root; while (u is InternalNode) { var internalNode = u as InternalNode; u = GetNode(internalNode.GetChildNodeId(key), profiler); } var l = u as LeafNode; return l.GetValue(key, valueBuff); } }
public void Delete(ulong txnId, byte[] key, BrightstarProfiler profiler) { using (profiler.Step("BPlusTree.Delete")) { if (_root is LeafNode) { (_root as LeafNode).Delete(key); MarkDirty(txnId, _root, profiler); } else { bool underAllocation; Delete(txnId, _root as InternalNode, key, out underAllocation, profiler); if (_root.KeyCount ==0) { // Now has only a single child leaf node, which should become the new tree root _root = GetNode((_root as InternalNode).ChildPointers[0], profiler); } } } }
private ulong Save(BrightstarProfiler profiler) { using (profiler.Step("Store.Save")) { _resourceTable.Commit(_currentTxnId + 1, profiler); var txnId = _currentTxnId + 1; var graphIndexId = _graphIndex.Save(txnId, profiler); var prefixManagerId = _prefixManager.Save(txnId, profiler); var resourceIndexId = _resourceIndex.Save(txnId, profiler); var subjectRelatedResourceIndexId = _subjectRelatedResourceIndex.Save(txnId, profiler); var objectRelatedResourceIndexId = _objectRelatedResourceIndex.Save(txnId, profiler); var buff = CreateStoreHeader(graphIndexId, prefixManagerId, resourceIndexId, subjectRelatedResourceIndexId, objectRelatedResourceIndexId); var page = _pageStore.Create(txnId); page.SetData(buff); page.SetData(buff, 0, 128); _pageStore.Commit(txnId, profiler); return page.Id; } }
public void Write(ulong commitId, ulong pageId, byte[] data, int srcOffset = 0, int pageOffset = 0, int len = -1, BrightstarProfiler profiler = null) { if (pageId < _newPageOffset) { throw new InvalidOperationException("Attempt to write to a fixed page"); } var pageIx = (int) (pageId - _newPageOffset); if (pageIx >= _newPages.Count) { throw new InvalidOperationException("Attempt to write to an unreserved page"); } using (profiler.Step("Write Page")) { var page = Retrieve(pageId, profiler); page.SetData(data, srcOffset, pageOffset, len); } }
/// <summary> /// Finds or creates a new ID for the graph with the specified graph URI /// </summary> /// <param name="graphUri">The graph URI to lookup</param> /// <param name="profiler"></param> /// <returns>The ID assigned to the graph</returns> public int AssertGraphId(string graphUri, BrightstarProfiler profiler = null) { if (String.IsNullOrEmpty(graphUri)) { throw new ArgumentException("Graph URI must not be null or an empty string", "graphUri"); } if (graphUri.Length > short.MaxValue) { throw new ArgumentException( String.Format("Graph URI string exceeds maximum allowed length of {0} bytes", short.MaxValue), "graphUri"); } #if WINDOWS_PHONE lock(_lock) { int entryId; if (_graphUriIndex.TryGetValue(graphUri, out entryId) && !_allEntries[entryId].IsDeleted) { return entryId; } var newId = _allEntries.Count; var entry = new GraphIndexEntry(newId, graphUri, false); _allEntries.Add(entry); _graphUriIndex.Add(graphUri, newId); return newId; } #else using (profiler.Step("Assert Graph Id")) { _lock.EnterUpgradeableReadLock(); try { int entryId; if (_graphUriIndex.TryGetValue(graphUri, out entryId) && !_allEntries[entryId].IsDeleted) { return entryId; } _lock.EnterWriteLock(); try { var newId = _allEntries.Count; var entry = new GraphIndexEntry(newId, graphUri, false); _allEntries.Add(entry); _graphUriIndex.Add(graphUri, newId); return newId; } finally { _lock.ExitWriteLock(); } } finally { _lock.ExitUpgradeableReadLock(); } } #endif }
public IEnumerable<KeyValuePair<byte[], byte []>> Scan(BrightstarProfiler profiler) { using (profiler.Step("Scan Entire BTree")) { return Scan(_root, profiler); } }
private void MarkDirty(ulong txnId, INode node, BrightstarProfiler profiler) { using (profiler.Step("MarkDirty")) { if (!node.IsDirty) { //_nodeCache.Remove(node.PageId); _nodeCache.Remove(node); if (!_pageStore.IsWriteable(node.PageId)) { node.PageId = _pageStore.Create(); } node.IsDirty = true; _modifiedNodes[node.PageId] = node; } _pageStore.Write(txnId, node.PageId, node.GetData(), profiler: profiler); //Task.Factory.StartNew(() => _pageStore.Write(txnId, node.PageId, node.GetData(), profiler:null)); // Not passing through the profiler because it is not thread-safe } }