public override Task OnPageAddedAsync(IPageStore store, Page page,
CancellationToken cancellationToken = default)
{
var pending = AsyncExtensions.TaskPool.Get();
try
{
var notify = _notify.Clients.All.SendAsync(Constants.Notifications.ReceiveMessage, "info",
$"Added new page with Title '{page.Title}' and ID '{page.Uuid}'", cancellationToken);
if (notify.IsCompleted || notify.IsCanceled || notify.IsFaulted)
{
return(AsyncExtensions.NoTask);
}
if (notify.Status != TaskStatus.Running)
{
notify.Start();
}
pending.Add(notify);
return(Task.WhenAll(pending));
}
finally
{
AsyncExtensions.TaskPool.Return(pending);
}
}
/// <summary>
/// Opens an existing tree in the page store
/// </summary>
/// <param name="pageStore"></param>
/// <param name="rootPageId">The page ID of the BTree root node</param>
/// <param name="keySize"></param>
/// <param name="dataSize"></param>
/// <param name="profiler"></param>
public BPlusTree(IPageStore pageStore, ulong rootPageId, int keySize = 8, int dataSize = 64, BrightstarProfiler profiler = null)
{
_config = new BPlusTreeConfiguration(pageStore, keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
var root = GetNode(rootPageId, profiler);
_rootId = root.PageId;
}
public PrefixManager(IPageStore pageStore, ulong startPageId, BrightstarProfiler profiler)
{
_pageStore = pageStore;
var startPage = _pageStore.Retrieve(startPageId, profiler);
Load(startPage, profiler);
}
public IStore OpenStore(string storeLocation, bool readOnly)
{
Logging.LogInfo("Open Store {0}", storeLocation);
var masterFile = GetMasterFile(storeLocation);
var latestCommitPoint = masterFile.GetLatestCommitPoint();
var dataFilePath = Path.Combine(storeLocation, DataFileName);
var resourceFilePath = Path.Combine(storeLocation, ResourceFileName);
if (_persistenceManager.FileExists(dataFilePath))
{
IPageStore dataPageStore = null;
switch (masterFile.PersistenceType)
{
case PersistenceType.AppendOnly:
dataPageStore = new AppendOnlyFilePageStore(_persistenceManager, dataFilePath, PageSize, readOnly, _storeConfiguration.DisableBackgroundWrites);
break;
case PersistenceType.Rewrite:
dataPageStore = new BinaryFilePageStore(_persistenceManager, dataFilePath, PageSize, readOnly,
latestCommitPoint.CommitNumber, latestCommitPoint.NextCommitNumber, _storeConfiguration.DisableBackgroundWrites);
break;
}
var resourcePageStore = new AppendOnlyFilePageStore(_persistenceManager, resourceFilePath, PageSize, readOnly, _storeConfiguration.DisableBackgroundWrites);
var resourceTable = new ResourceTable(resourcePageStore);
var store = new Store(storeLocation, dataPageStore, resourceTable, latestCommitPoint.LocationOffset, null);
Logging.LogInfo("Store {0} opened successfully", storeLocation);
return(store);
}
throw new StoreManagerException(storeLocation, "Data file not found");
}
public void Setup()
{
_startupLog = SetupMock <IStartupLog>();
_errorLog = SetupMock <IErrorLog>();
_pagePoolFactory = SetupMock <IPagePoolFactory>();
_pagePool = _pagePoolFactory.Create(_pageSize);
_logFileInfo1 = new FileInfo("C:\\temp\\test1.ldf");
_logFileInfo2 = new FileInfo("C:\\temp\\test2.ldf");
_dataFileInfo1 = new FileInfo("C:\\temp\\test1.mdf");
_dataFileInfo2 = new FileInfo("C:\\temp\\test2.mdf");
_fileSet = new FileSet(
new IDataFile[] { new DataFile(_dataFileInfo1, _pageSize, _startupLog), new DataFile(_dataFileInfo2, _pageSize, _startupLog) },
new ILogFile[] { new LogFile(_logFileInfo1, true, _startupLog), new LogFile(_logFileInfo2, true, _startupLog) },
_pagePoolFactory,
_startupLog);
var databaseFactory = SetupMock <IDatabaseFactory>();
_database = databaseFactory.Open(null);
var pageCache = new PageCache(_fileSet, _database, _pagePoolFactory, _startupLog, _errorLog);
_pageStore = new PageStore(pageCache, _startupLog);
}
public void Setup()
{
_startUpLog = SetupMock <IStartupLog>();
_errorLog = SetupMock <IErrorLog>();
_pagePoolFactory = SetupMock <IPagePoolFactory>();
_pagePool = _pagePoolFactory.Create(_pageSize);
_dataFileInfo = new FileInfo("C:\\temp\\test.mdf");
_dataFile = new DataFile(_dataFileInfo, _pageSize, _startUpLog);
_logFileInfo = new FileInfo("C:\\temp\\test.ldf");
_logFile = new LogFile(_logFileInfo, true, _startUpLog);
_fileSet = new FileSet(
new[] { _dataFile },
new[] { _logFile },
_pagePoolFactory,
_startUpLog);
var databaseFactory = SetupMock <IDatabaseFactory>();
_database = databaseFactory.Open(null);
var pageCache = new PageCache(_fileSet, _database, _pagePoolFactory, _startUpLog, _errorLog);
_pageStore = new PageStore(pageCache, _startUpLog);
_accessorFactory = new AccessorFactory();
_accessor = _accessorFactory.SmallSequentialAccessor(_pageStore);
}
/// <summary>
/// Creates a new tree in the page store
/// </summary>
/// <param name="txnId">The transaction id for the update</param>
/// <param name="pageStore"></param>
/// <param name="keySize">The size of the B+ tree's key (in bytes)</param>
/// <param name="dataSize">The size of the values stored in leaf nodes (in bytes)</param>
public BPlusTree(ulong txnId, IPageStore pageStore, int keySize = 8, int dataSize = 64)
{
_config = new BPlusTreeConfiguration(pageStore, keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
var root = MakeLeafNode(txnId);
_rootId = root.PageId;
}
public BPlusTreeBuilder(IPageStore targetPageStore, BPlusTreeConfiguration targetTreeConfiguration)
{
_pageStore = targetPageStore;
_config = targetTreeConfiguration;
// These values are copied locally to allow us to later change the default loading from 100% to something lower if we want
_leafLoadFactor = _config.LeafLoadFactor;
_internalBranchFactor = _config.InternalBranchFactor;
}
public BPlusTreeBuilder(IPageStore targetPageStore, BPlusTreeConfiguration targetTreeConfiguration)
{
_pageStore = targetPageStore;
_config = targetTreeConfiguration;
// These values are copied locally to allow us to later change the default loading from 100% to something lower if we want
_leafLoadFactor = _config.LeafLoadFactor;
_internalBranchFactor = _config.InternalBranchFactor;
}
/// <summary>
/// Opens an existing tree in the page store
/// </summary>
/// <param name="pageStore"></param>
/// <param name="rootPageId">The page ID of the BTree root node</param>
/// <param name="keySize"></param>
/// <param name="dataSize"></param>
/// <param name="profiler"></param>
public BPlusTree(IPageStore pageStore, ulong rootPageId, int keySize = 8, int dataSize = 64, BrightstarProfiler profiler = null)
{
_config = new BPlusTreeConfiguration(pageStore, keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
var root = GetNode(rootPageId, profiler);
_rootId = root.PageId;
}
/// <summary>
/// Creates a new tree in the page store
/// </summary>
/// <param name="txnId">The transaction id for the update</param>
/// <param name="pageStore"></param>
/// <param name="keySize">The size of the B+ tree's key (in bytes)</param>
/// <param name="dataSize">The size of the values stored in leaf nodes (in bytes)</param>
public BPlusTree(ulong txnId, IPageStore pageStore, int keySize = 8, int dataSize = 64)
{
_config = new BPlusTreeConfiguration(pageStore, keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
var root = MakeLeafNode(txnId);
_rootId = root.PageId;
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
var targetConfiguration = new BPlusTreeConfiguration(pageStore, Configuration.KeySize,
Configuration.ValueSize, Configuration.PageSize);
var builder = new BPlusTreeBuilder(pageStore, targetConfiguration);
return(builder.Build(transactionId, Scan(profiler), profiler));
}
public void CreateSnapshot(string srcStoreLocation, string destStoreLocation,
PersistenceType storePersistenceType, ulong commitPointId = StoreConstants.NullUlong)
{
Logging.LogInfo("Snapshot store {0} to new store {1} with persistence type {2}", srcStoreLocation,
destStoreLocation, storePersistenceType);
if (_persistenceManager.DirectoryExists(destStoreLocation))
{
throw new StoreManagerException(destStoreLocation, "Store already exists");
}
// Open the source store for reading
using (IStore srcStore = commitPointId == StoreConstants.NullUlong
? OpenStore(srcStoreLocation, true)
: OpenStore(srcStoreLocation, commitPointId))
{
// Create the directory for the destination store
_persistenceManager.CreateDirectory(destStoreLocation);
// Create empty data file
var dataFilePath = Path.Combine(destStoreLocation, DataFileName);
_persistenceManager.CreateFile(dataFilePath);
// Create initial master file
var destStoreConfiguration = _storeConfiguration.Clone() as StoreConfiguration;
destStoreConfiguration.PersistenceType = storePersistenceType;
var destMasterFile = MasterFile.Create(_persistenceManager, destStoreLocation, destStoreConfiguration,
Guid.NewGuid());
// Copy resource files from source store
var resourceFilePath = Path.Combine(destStoreLocation, ResourceFileName);
_persistenceManager.CopyFile(Path.Combine(srcStoreLocation, ResourceFileName), resourceFilePath, true);
// Initialize data page store
IPageStore destPageStore = null;
switch (storePersistenceType)
{
case PersistenceType.AppendOnly:
destPageStore = new AppendOnlyFilePageStore(_persistenceManager, dataFilePath, PageSize, false,
_storeConfiguration.DisableBackgroundWrites);
break;
case PersistenceType.Rewrite:
destPageStore = new BinaryFilePageStore(_persistenceManager, dataFilePath, PageSize, false, 0, 1, _storeConfiguration.DisableBackgroundWrites);
break;
default:
throw new BrightstarInternalException("Unrecognized target store type: " + storePersistenceType);
}
// Copy Data
ulong destStorePageId = srcStore.CopyTo(destPageStore, 1ul);
destPageStore.Close();
destMasterFile.AppendCommitPoint(
new CommitPoint(destStorePageId, 1ul, DateTime.UtcNow, Guid.Empty), true);
}
}
/// <summary>
/// Creates a new tree in the page store
/// </summary>
/// <param name="txnId">The transaction id for the update</param>
/// <param name="pageStore"></param>
/// <param name="keySize">The size of the B+ tree's key (in bytes)</param>
/// <param name="dataSize">The size of the values stored in leaf nodes (in bytes)</param>
public BPlusTree(ulong txnId, IPageStore pageStore, int keySize = 8, int dataSize = 64)
{
_config = new BPlusTreeConfiguration(pageStore, keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
var root = MakeLeafNode(txnId);
_rootId = root.PageId;
_nodeCache = new WeakReferenceNodeCache();
_nodeCache.Add(root);
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
using (profiler.Step("RelatedResourceIndex.Write"))
{
var indexBuilder = new BPlusTreeBuilder(pageStore, Configuration);
return(indexBuilder.Build(transactionId, WritePredicateIndexes(pageStore, transactionId, profiler),
profiler));
}
}
public ResourceTable(IPageStore pageStore)
{
_pageStore = pageStore;
_segmentSize = CalculateSegmentSize(_pageStore.PageSize);
_pointerSegment = (byte)((_pageStore.PageSize - 8) / _segmentSize);
_writeLock = new object();
_nextSegment = _pointerSegment;
_currentPage = 0;
}
/// <summary>
/// Opens an existing tree in the page store
/// </summary>
/// <param name="pageStore"></param>
/// <param name="rootPageId">The page ID of the BTree root node</param>
/// <param name="keySize"></param>
/// <param name="dataSize"></param>
/// <param name="profiler"></param>
public BPlusTree(IPageStore pageStore, ulong rootPageId, int keySize = 8, int dataSize = 64, BrightstarProfiler profiler = null)
{
_config = new BPlusTreeConfiguration(keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
_modifiedNodes = new Dictionary<ulong, INode>();
_nodeCache = new WeakReferenceNodeCache();
_root = GetNode(rootPageId, profiler);
_nodeCache.Add(_root);
}
public PageService(
IPageStore pageStore,
ICultureStore cultureStore,
IMapper mapper)
{
_pageStore = pageStore;
_cultureStore = cultureStore;
_mapper = mapper;
}
public ResourceTable(IPageStore pageStore)
{
_pageStore = pageStore;
_segmentSize = CalculateSegmentSize(_pageStore.PageSize);
_pointerSegment = (byte) ((_pageStore.PageSize - 8)/_segmentSize);
_writeLock = new object();
_nextSegment = _pointerSegment;
_currentPage = 0;
}
/// <summary>
/// Opens an existing tree in the page store
/// </summary>
/// <param name="pageStore"></param>
/// <param name="rootPageId">The page ID of the BTree root node</param>
/// <param name="keySize"></param>
/// <param name="dataSize"></param>
/// <param name="profiler"></param>
public BPlusTree(IPageStore pageStore, ulong rootPageId, int keySize = 8, int dataSize = 64, BrightstarProfiler profiler = null)
{
_config = new BPlusTreeConfiguration(keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
_modifiedNodes = new Dictionary <ulong, INode>();
_nodeCache = new WeakReferenceNodeCache();
_root = GetNode(rootPageId, profiler);
_nodeCache.Add(_root);
}
public ConcurrentGraphIndex(IPageStore pageStore, ulong rootPage, BrightstarProfiler profiler)
{
using (profiler.Step("Load ConcurrentGraphIndex"))
{
_pageStore = pageStore;
_graphUriIndex = new Dictionary <string, int>();
_allEntries = new List <GraphIndexEntry>();
Read(rootPage, profiler);
}
}
/// <summary>
/// Creates a new tree in the page store
/// </summary>
/// <param name="txnId">The transaction id for the update</param>
/// <param name="pageStore"></param>
/// <param name="keySize">The size of the B+ tree's key (in bytes)</param>
/// <param name="dataSize">The size of the values stored in leaf nodes (in bytes)</param>
public BPlusTree(ulong txnId, IPageStore pageStore, int keySize = 8, int dataSize = 64)
{
_config = new BPlusTreeConfiguration(pageStore, keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
var root = MakeLeafNode(txnId);
_rootId = root.PageId;
_nodeCache = new WeakReferenceNodeCache();
_nodeCache.Add(root);
}
public ConcurrentGraphIndex(IPageStore pageStore, ulong rootPage, BrightstarProfiler profiler)
{
using (profiler.Step("Load ConcurrentGraphIndex"))
{
_pageStore = pageStore;
_graphUriIndex = new Dictionary<string, int>();
_allEntries = new List<GraphIndexEntry>();
Read(rootPage, profiler);
}
}
/// <summary>
/// Creates a new tree in the page store
/// </summary>
/// <param name="pageStore"></param>
/// <param name="keySize">The size of the B+ tree's key (in bytes)</param>
/// <param name="dataSize">The size of the values stored in leaf nodes (in bytes)</param>
public BPlusTree(IPageStore pageStore, int keySize = 8, int dataSize = 64)
{
_config = new BPlusTreeConfiguration(keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
_modifiedNodes = new Dictionary <ulong, INode>();
_root = new LeafNode(pageStore.Create(), 0, 0, _config);
_nodeCache = new WeakReferenceNodeCache();
_modifiedNodes[_root.PageId] = _root;
_nodeCache.Add(_root);
}
/// <summary>
/// Creates a new tree in the page store
/// </summary>
/// <param name="pageStore"></param>
/// <param name="keySize">The size of the B+ tree's key (in bytes)</param>
/// <param name="dataSize">The size of the values stored in leaf nodes (in bytes)</param>
public BPlusTree(IPageStore pageStore, int keySize = 8, int dataSize = 64)
{
_config = new BPlusTreeConfiguration(keySize, dataSize, pageStore.PageSize);
_pageStore = pageStore;
_modifiedNodes = new Dictionary<ulong, INode>();
_root = new LeafNode(pageStore.Create(), 0, 0, _config);
_nodeCache = new WeakReferenceNodeCache();
_modifiedNodes[_root.PageId] = _root;
_nodeCache.Add(_root);
}
public void Setup()
{
var databaseFactory = SetupMock <IDatabaseFactory>();
_database = databaseFactory.Open(null);
_pageStore = SetupMock <IPageStore>();
_accessorFactory = new AccessorFactory();
}
private IEnumerable <KeyValuePair <byte[], byte []> > WritePredicateIndexes(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
foreach (var entry in EnumeratePredicateIndexes(profiler))
{
var predicateId = entry.Key;
var builder = new BPlusTreeBuilder(pageStore, entry.Value.Configuration);
ulong newPredicateIndexId = builder.Build(transactionId, entry.Value.Scan(profiler), profiler);
yield return(new KeyValuePair <byte[], byte[]>(BitConverter.GetBytes(predicateId), BitConverter.GetBytes(newPredicateIndexId)));
}
}
public Store(string storeLocation, IPageStore dataPageStore, IResourceTable resourceTable)
{
DirectoryPath = storeLocation;
_pageStore = dataPageStore;
_graphIndex = new ConcurrentGraphIndex(_pageStore);
_subjectRelatedResourceIndex = new RelatedResourceIndex.RelatedResourceIndex(_currentTxnId + 1, _pageStore);
_objectRelatedResourceIndex = new RelatedResourceIndex.RelatedResourceIndex(_currentTxnId + 1, _pageStore);
_prefixManager = new PrefixManager(_pageStore);
_resourceTable = resourceTable;
_resourceIndex = new ResourceIndex.ResourceIndex(1, _pageStore, _resourceTable);
}
public Store(string storeLocation, IPageStore dataPageStore, IResourceTable resourceTable)
{
DirectoryPath = storeLocation;
_pageStore = dataPageStore;
_graphIndex = new ConcurrentGraphIndex(_pageStore);
_subjectRelatedResourceIndex = new RelatedResourceIndex.RelatedResourceIndex(_currentTxnId + 1, _pageStore);
_objectRelatedResourceIndex = new RelatedResourceIndex.RelatedResourceIndex(_currentTxnId + 1, _pageStore);
_prefixManager = new PrefixManager(_pageStore);
_resourceTable = resourceTable;
_resourceIndex = new ResourceIndex.ResourceIndex(1, _pageStore, _resourceTable);
}
public Store(string storeLocation, IPageStore dataPageStore, IResourceTable resourceTable, ulong storePageId, BrightstarProfiler profiler)
{
using (profiler.Step("Load Store"))
{
DirectoryPath = storeLocation;
_pageStore = dataPageStore;
_resourceTable = resourceTable;
var storePage = _pageStore.Retrieve(storePageId, profiler);
Load(storePage, profiler);
}
}
public Store(string storeLocation, IPageStore dataPageStore, IResourceTable resourceTable, ulong storePageId, BrightstarProfiler profiler)
{
using (profiler.Step("Load Store"))
{
DirectoryPath = storeLocation;
_pageStore = dataPageStore;
_resourceTable = resourceTable;
var storePage = _pageStore.Retrieve(storePageId, profiler);
Load(storePage, profiler);
}
}
/// <summary>
/// Opens an existing resource index from the specified page store
/// </summary>
/// <param name="pageStore"></param>
/// <param name="resourceTable">The table used to store long resource strings</param>
/// <param name="rootNodeId">The ID of the page that contains the root node of the resource index</param>
public ResourceIndex(IPageStore pageStore, IResourceTable resourceTable, ulong rootNodeId) : base(pageStore, rootNodeId)
{
//_resourceCache = new ConcurrentResourceCache();
//_resourceIdCache = new ConcurrentResourceIdCache();
_resourceCache= new LruResourceCache();
_resourceIdCache = new LruResourceIdCache();
_resourceStore = new ResourceStore(resourceTable);
#if DEBUG_BTREE
Configuration.DebugId = "ResIx";
Logging.LogDebug("Opened new {0} BTree with root page {1}", Configuration.DebugId, rootNodeId);
#endif
}
/// <summary>
/// Creates a new empty resource index in the specified page store
/// </summary>
/// <param name="txnId"></param>
/// <param name="pageStore"></param>
/// <param name="resourceTable"></param>
public ResourceIndex(ulong txnId, IPageStore pageStore, IResourceTable resourceTable) : base(txnId, pageStore)
{
//_resourceCache = new ConcurrentResourceCache();
//_resourceIdCache = new ConcurrentResourceIdCache();
_resourceCache = new LruResourceCache();
_resourceIdCache = new LruResourceIdCache();
_resourceStore = new ResourceStore(resourceTable);
#if DEBUG_BTREE
Configuration.DebugId = "ResIx";
Logging.LogDebug("Created new {0} BTree with root page {1}", Configuration.DebugId, RootId);
#endif
}
public IStore CreateStore(string storeLocation, PersistenceType storePersistenceType, bool readOnly, bool withTransactionLogging)
{
Logging.LogInfo("Create Store {0} with persistence type {1}", storeLocation, storePersistenceType);
if (_persistenceManager.DirectoryExists(storeLocation))
{
throw new StoreManagerException(storeLocation, "Store already exists");
}
_persistenceManager.CreateDirectory(storeLocation);
var dataFilePath = Path.Combine(storeLocation, DataFileName);
_persistenceManager.CreateFile(dataFilePath);
var resourceFilePath = Path.Combine(storeLocation, ResourceFileName);
_persistenceManager.CreateFile(resourceFilePath);
var targetStoreConfiguration = _storeConfiguration.Clone() as StoreConfiguration;
targetStoreConfiguration.PersistenceType = storePersistenceType;
MasterFile.Create(_persistenceManager, storeLocation, targetStoreConfiguration, Guid.NewGuid());
IPageStore dataPageStore = null;
switch (storePersistenceType)
{
case PersistenceType.AppendOnly:
dataPageStore = new AppendOnlyFilePageStore(_persistenceManager, dataFilePath, PageSize, false, _storeConfiguration.DisableBackgroundWrites);
break;
case PersistenceType.Rewrite:
dataPageStore = new BinaryFilePageStore(_persistenceManager, dataFilePath, PageSize, false, 0, 1, _storeConfiguration.DisableBackgroundWrites);
break;
}
IPageStore resourcePageStore = new AppendOnlyFilePageStore(_persistenceManager, resourceFilePath, PageSize, false, _storeConfiguration.DisableBackgroundWrites);
var resourceTable = new ResourceTable(resourcePageStore);
using (var store = new Store(storeLocation, dataPageStore, resourceTable))
{
store.Commit(Guid.Empty);
store.Close();
}
if (withTransactionLogging)
{
_persistenceManager.CreateFile(Path.Combine(storeLocation, "transactionheaders.bs"));
_persistenceManager.CreateFile(Path.Combine(storeLocation, "transactions.bs"));
}
Logging.LogInfo("Store created at {0}", storeLocation);
return(OpenStore(storeLocation, readOnly));
}
/// <summary>
/// Creates a new empty resource index in the specified page store
/// </summary>
/// <param name="txnId"></param>
/// <param name="pageStore"></param>
/// <param name="resourceTable"></param>
public ResourceIndex(ulong txnId, IPageStore pageStore, IResourceTable resourceTable) : base(txnId, pageStore)
{
//_resourceCache = new ConcurrentResourceCache();
//_resourceIdCache = new ConcurrentResourceIdCache();
_resourceCache = new LruResourceCache();
_resourceIdCache = new LruResourceIdCache();
_resourceStore = new ResourceStore(resourceTable);
#if DEBUG_BTREE
Configuration.DebugId = "ResIx";
Logging.LogDebug("Created new {0} BTree with root page {1}", Configuration.DebugId, RootId);
#endif
}
/// <summary>
/// Opens an existing resource index from the specified page store
/// </summary>
/// <param name="pageStore"></param>
/// <param name="resourceTable">The table used to store long resource strings</param>
/// <param name="rootNodeId">The ID of the page that contains the root node of the resource index</param>
public ResourceIndex(IPageStore pageStore, IResourceTable resourceTable, ulong rootNodeId) : base(pageStore, rootNodeId)
{
//_resourceCache = new ConcurrentResourceCache();
//_resourceIdCache = new ConcurrentResourceIdCache();
_resourceCache = new LruResourceCache();
_resourceIdCache = new LruResourceIdCache();
_resourceStore = new ResourceStore(resourceTable);
#if DEBUG_BTREE
Configuration.DebugId = "ResIx";
Logging.LogDebug("Opened new {0} BTree with root page {1}", Configuration.DebugId, rootNodeId);
#endif
}
public BPlusTreeConfiguration(IPageStore pageStore, int keySize, int valueSize, int pageSize)
{
PageStore = pageStore;
KeySize = keySize;
ValueSize = valueSize;
PageSize = pageSize;
InternalBranchFactor = (PageSize - InternalNodeHeaderSize - NodePointerSize) / (KeySize + NodePointerSize);
InternalSplitIndex = InternalBranchFactor / 2;
InternalNodeChildStartOffset = InternalNodeHeaderSize + (InternalBranchFactor * KeySize);
LeafLoadFactor = (PageSize - LeafNodeHeaderSize) / (KeySize + ValueSize + 1); // Each value requires ValueSize+1 bytes, extra byte is for the value length
LeafSplitIndex = LeafLoadFactor / 2;
LeafDataStartOffset = LeafNodeHeaderSize + (KeySize * LeafLoadFactor);
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
IPage rootPage = pageStore.Create(transactionId);
IPage currentPage = rootPage;
var buff = new byte[pageStore.PageSize];
int offset = 0;
foreach (var graphIndexEntry in _allEntries)
{
int entrySize = String.IsNullOrEmpty(graphIndexEntry.Uri)
? 1
: 3 + Encoding.UTF8.GetByteCount(graphIndexEntry.Uri);
if (offset + entrySize > pageStore.PageSize - 9)
{
IPage nextPage = pageStore.Create(transactionId);
buff[offset] = 0xff;
BitConverter.GetBytes(nextPage.Id).CopyTo(buff, pageStore.PageSize - 8);
currentPage.SetData(buff);
currentPage = nextPage;
offset = 0;
}
else
{
if (String.IsNullOrEmpty(graphIndexEntry.Uri))
{
// Record an empty entry
buff[offset++] = 2;
}
else
{
if (graphIndexEntry.IsDeleted)
{
buff[offset++] = 1;
}
else
{
buff[offset++] = 0;
}
var uriBytes = Encoding.UTF8.GetBytes(graphIndexEntry.Uri);
BitConverter.GetBytes(uriBytes.Length).CopyTo(buff, offset);
offset += 4;
uriBytes.CopyTo(buff, offset);
offset += uriBytes.Length;
}
}
}
buff[offset] = 0xff;
BitConverter.GetBytes(0ul).CopyTo(buff, pageStore.PageSize - 8);
currentPage.SetData(buff);
return(rootPage.Id);
}
/// <summary>
/// Copies all the indexes from this store to the specified target page store
/// </summary>
/// <param name="pageStore">The page store to copy to</param>
/// <param name="txnId">The transaction Id to use in the target page store for the write</param>
/// <returns>The ID of the root store page in the target page store</returns>
public ulong CopyTo(IPageStore pageStore, ulong txnId)
{
var graphIndexId = _graphIndex.Write(pageStore, txnId, null);
var prefixManagerId = _prefixManager.Write(pageStore, txnId, null);
var resourceIndexId = _resourceIndex.Write(pageStore, txnId, null);
var subjectRelatedResourceIndexId = _subjectRelatedResourceIndex.Write(pageStore, txnId, null);
var objectRelatedResourceIndexId = _objectRelatedResourceIndex.Write(pageStore, txnId, null);
var buff = CreateStoreHeader(graphIndexId, prefixManagerId, resourceIndexId, subjectRelatedResourceIndexId,
objectRelatedResourceIndexId);
var storePage = pageStore.Create(txnId);
storePage.SetData(buff);
storePage.SetData(buff, 0, 128);
pageStore.Commit(txnId, null);
return(storePage.Id);
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
IPage startPage = pageStore.Create(transactionId);
IPage currentPage = startPage;
byte[] buff = new byte[pageStore.PageSize];
int offset = 0;
foreach (var entry in _shortValueMappings)
{
byte[] encodedPrefix = Encoding.UTF8.GetBytes(entry.Key);
byte[] encodedUri = Encoding.UTF8.GetBytes(entry.Value);
int totalLength = encodedUri.Length + encodedPrefix.Length + 4;
if (offset + totalLength > (pageStore.PageSize - 10))
{
// Not enough room for the entry and the next page pointer
// So create a new page for this entry and write a pointer to it
// onto the current page
IPage nextPage = pageStore.Create(transactionId);
BitConverter.GetBytes(ushort.MaxValue).CopyTo(buff, offset);
offset += 2;
BitConverter.GetBytes(nextPage.Id).CopyTo(buff, offset);
currentPage.SetData(buff);
currentPage = nextPage;
buff = new byte[pageStore.PageSize];
offset = 0;
}
BitConverter.GetBytes((ushort)encodedPrefix.Length).CopyTo(buff, offset);
offset += 2;
encodedPrefix.CopyTo(buff, offset);
offset += encodedPrefix.Length;
BitConverter.GetBytes((ushort)encodedUri.Length).CopyTo(buff, offset);
offset += 2;
encodedUri.CopyTo(buff, offset);
offset += encodedUri.Length;
}
// Write the end marker
BitConverter.GetBytes(ushort.MaxValue).CopyTo(buff, offset);
offset += 2;
BitConverter.GetBytes(0ul).CopyTo(buff, offset);
currentPage.SetData(buff);
return(startPage.Id);
}
private void BuildAndScan(IPageStore pageStore, string pageStoreName, PersistenceType persistenceType, int keyCount)
{
var config = new BPlusTreeConfiguration(8, 64, pageStore.PageSize);
var builder = new BPlusTreeBuilder(pageStore, config);
var treeRoot = builder.Build(1, _testOrderedValues.Take(keyCount));
var treeBeforeSave = new BPlusTree(pageStore, treeRoot);
treeBeforeSave.Save(1, null);
//ValidateScan(treeBeforeSave.Scan(null), keyCount, pageStoreName + " before save");
pageStore.Commit(1, null);
pageStore.Close();
using(var ps = TestUtils.OpenPageStore(pageStoreName, true, persistenceType, 1))
{
var tree = new BPlusTree(ps, treeRoot);
tree.DumpStructure();
ValidateScan(tree.Scan(null), keyCount, pageStoreName + " after save");
ValidateSearch(tree, keyCount, pageStoreName + "after save");
}
}
private void BuildAndScan(IPageStore pageStore, string pageStoreName, PersistenceType persistenceType, int keyCount)
{
var config = new BPlusTreeConfiguration(8, 64, pageStore.PageSize);
var builder = new BPlusTreeBuilder(pageStore, config);
var treeRoot = builder.Build(1, _testOrderedValues.Take(keyCount));
var treeBeforeSave = new BPlusTree(pageStore, treeRoot);
treeBeforeSave.Save(1, null);
//ValidateScan(treeBeforeSave.Scan(null), keyCount, pageStoreName + " before save");
pageStore.Commit(1, null);
pageStore.Close();
using (var ps = TestUtils.OpenPageStore(pageStoreName, true, persistenceType, 1))
{
var tree = new BPlusTree(ps, treeRoot);
tree.DumpStructure();
ValidateScan(tree.Scan(null), keyCount, pageStoreName + " after save");
ValidateSearch(tree, keyCount, pageStoreName + "after save");
}
}
public static void Main(string[] args)
{
Application.Initialize(ConfigurationHelper.ConfigureDependencies);
try
{
Store = Application.Current.Container.GetInstance <IPageStore>();
var injectedEngine = Application.Current.Container.GetInstance <IEngine>();
var basketPage = Store.Pages.OfType <BasketPage>().FirstOrDefault();
var registerPage = Store.Pages.OfType <RegisterPage>().FirstOrDefault();
var basketController = new BasketController(basketPage, injectedEngine);
var registerController = new RegisterController(registerPage, injectedEngine);
basketController.Index();
registerController.Index();
Console.ReadKey();
}
catch (Exception ex)
{
Console.WriteLine("There was an exception of {0} type", ex.GetType());
}
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
using (profiler.Step("RelatedResourceIndex.Write"))
{
var targetConfiguration = new BPlusTreeConfiguration(pageStore, Configuration.KeySize,
Configuration.ValueSize, Configuration.PageSize);
var indexBuilder = new BPlusTreeBuilder(pageStore, targetConfiguration);
return indexBuilder.Build(transactionId, WritePredicateIndexes(pageStore, transactionId, profiler),
profiler);
}
}
private IEnumerable<KeyValuePair<byte[], byte []>> WritePredicateIndexes(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
foreach (var entry in EnumeratePredicateIndexes(profiler))
{
var predicateId = entry.Key;
var targetConfiguration = new BPlusTreeConfiguration(pageStore, entry.Value.Configuration.KeySize,
entry.Value.Configuration.ValueSize,
entry.Value.Configuration.PageSize);
var builder = new BPlusTreeBuilder(pageStore, targetConfiguration);
ulong newPredicateIndexId = builder.Build(transactionId, entry.Value.Scan(profiler), profiler);
yield return new KeyValuePair<byte[], byte[]>(BitConverter.GetBytes(predicateId), BitConverter.GetBytes(newPredicateIndexId));
}
}
public PredicateRelatedResourceIndex(IPageStore pageStore, ulong rootPageId) : base(pageStore, rootPageId, RelatedResourceIndex.KeySize, 0){}
public PredicateRelatedResourceIndex(IPageStore pageStore) : base(pageStore, RelatedResourceIndex.KeySize, 0){}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
var builder = new BPlusTreeBuilder(pageStore, Configuration);
return builder.Build(transactionId, Scan(profiler), profiler);
}
public RelatedResourceIndex(ulong txnId, IPageStore pageStore): base(txnId, pageStore, 8, 8)
{
_predicateIndexes = new Dictionary<ulong, PredicateRelatedResourceIndex>();
}
/// <summary>
/// Copies all the indexes from this store to the specified target page store
/// </summary>
/// <param name="pageStore">The page store to copy to</param>
/// <param name="txnId">The transaction Id to use in the target page store for the write</param>
/// <returns>The ID of the root store page in the target page store</returns>
public ulong CopyTo(IPageStore pageStore, ulong txnId)
{
var graphIndexId = _graphIndex.Write(pageStore, txnId, null);
var prefixManagerId = _prefixManager.Write(pageStore, txnId, null);
var resourceIndexId = _resourceIndex.Write(pageStore, txnId, null);
var subjectRelatedResourceIndexId = _subjectRelatedResourceIndex.Write(pageStore, txnId, null);
var objectRelatedResourceIndexId = _objectRelatedResourceIndex.Write(pageStore, txnId, null);
var buff = CreateStoreHeader(graphIndexId, prefixManagerId, resourceIndexId, subjectRelatedResourceIndexId,
objectRelatedResourceIndexId);
var storePage = pageStore.Create(txnId);
storePage.SetData(buff);
storePage.SetData(buff, 0, 128);
pageStore.Commit(txnId, null);
return storePage.Id;
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
var targetConfiguration = new BPlusTreeConfiguration(pageStore, Configuration.KeySize,
Configuration.ValueSize, Configuration.PageSize);
var builder = new BPlusTreeBuilder(pageStore, targetConfiguration);
return builder.Build(transactionId, Scan(profiler), profiler);
}
private void TestBuildSingleLeafNode(IPageStore pageStore, string pageStoreName, PersistenceType persistenceType)
{
BuildAndScan(pageStore, pageStoreName, persistenceType, 20);
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
ulong startPageId = pageStore.Create();
ulong currentPageId = startPageId;
byte[] currentPage = new byte[pageStore.PageSize];
int offset = 0;
foreach (var entry in _shortValueMappings)
{
byte[] encodedPrefix = Encoding.UTF8.GetBytes(entry.Key);
byte[] encodedUri = Encoding.UTF8.GetBytes(entry.Value);
int totalLength = encodedUri.Length + encodedPrefix.Length + 4;
if (offset + totalLength > (pageStore.PageSize - 10))
{
// Not enough room for the entry and the next page pointer
// So create a new page for this entry and write a pointer to it
// onto the current page
ulong nextPage = pageStore.Create();
BitConverter.GetBytes(ushort.MaxValue).CopyTo(currentPage, offset);
offset += 2;
BitConverter.GetBytes(nextPage).CopyTo(currentPage, offset);
pageStore.Write(transactionId, currentPageId, currentPage, profiler: profiler);
currentPageId = nextPage;
currentPage = new byte[pageStore.PageSize];
offset = 0;
}
BitConverter.GetBytes((ushort)encodedPrefix.Length).CopyTo(currentPage, offset);
offset += 2;
encodedPrefix.CopyTo(currentPage, offset);
offset += encodedPrefix.Length;
BitConverter.GetBytes((ushort)encodedUri.Length).CopyTo(currentPage, offset);
offset += 2;
encodedUri.CopyTo(currentPage, offset);
offset += encodedUri.Length;
}
// Write the end marker
BitConverter.GetBytes(ushort.MaxValue).CopyTo(currentPage, offset);
offset += 2;
BitConverter.GetBytes(0ul).CopyTo(currentPage, offset);
pageStore.Write(transactionId, currentPageId, currentPage, profiler: profiler);
return startPageId;
}
/// <summary>
/// Creates a new prefix manager that will write its content to the specified page store
/// </summary>
/// <param name="pageStore"></param>
public PrefixManager(IPageStore pageStore)
{
_pageStore = pageStore;
}
public PrefixManager(IPageStore pageStore, ulong startPageId, BrightstarProfiler profiler)
{
_pageStore = pageStore;
var startPage = _pageStore.Retrieve(startPageId, profiler);
Load(startPage, profiler);
}
public PagesController(IPageStore pageStore)
{
this.pageStore = pageStore;
}
public RelatedResourceIndex(IPageStore pageStore, ulong rootPageId, BrightstarProfiler profiler) : base(pageStore, rootPageId, 8, 8, profiler)
{
_predicateIndexes = new Dictionary<ulong, PredicateRelatedResourceIndex>();
}
public PredicateRelatedResourceIndex(ulong txnId, IPageStore pageStore) : base(txnId, pageStore, RelatedResourceIndex.KeySize, 0){}
public ConcurrentGraphIndex(IPageStore pageStore)
{
_pageStore = pageStore;
_graphUriIndex = new Dictionary<string, int>();
_allEntries= new List<GraphIndexEntry>();
}
public ulong Write(IPageStore pageStore, ulong transactionId, BrightstarProfiler profiler)
{
ulong rootPage = pageStore.Create();
ulong currentPage = rootPage;
var buff = new byte[pageStore.PageSize];
int offset = 0;
foreach (var graphIndexEntry in _allEntries)
{
int entrySize = String.IsNullOrEmpty(graphIndexEntry.Uri)
? 1
: 3 + Encoding.UTF8.GetByteCount(graphIndexEntry.Uri);
if (offset + entrySize > pageStore.PageSize - 9)
{
ulong nextPage = pageStore.Create();
buff[offset] = 0xff;
BitConverter.GetBytes(nextPage).CopyTo(buff, pageStore.PageSize - 8);
pageStore.Write(transactionId, currentPage, buff, profiler: profiler);
currentPage = nextPage;
offset = 0;
}
else
{
if (String.IsNullOrEmpty(graphIndexEntry.Uri))
{
// Record an empty entry
buff[offset++] = 2;
}
else
{
if (graphIndexEntry.IsDeleted)
{
buff[offset++] = 1;
}
else
{
buff[offset++] = 0;
}
var uriBytes = Encoding.UTF8.GetBytes(graphIndexEntry.Uri);
BitConverter.GetBytes(uriBytes.Length).CopyTo(buff, offset);
offset += 4;
uriBytes.CopyTo(buff, offset);
offset += uriBytes.Length;
}
}
}
buff[offset] = 0xff;
BitConverter.GetBytes(0ul).CopyTo(buff, pageStore.PageSize - 8);
pageStore.Write(transactionId, currentPage, buff, profiler: profiler);
return rootPage;
}
