/// <summary>
/// Removes all unused data.
/// </summary>
/// <param name="jobId"></param>
public void Consolidate(Guid jobId)
{
var storeManager = StoreManagerFactory.GetStoreManager();
var consolidatePageStore = storeManager.CreateConsolidationStore(DirectoryPath);
ulong txnId = _currentTxnId + 1;
var graphIndexId = _graphIndex.Write(consolidatePageStore, txnId, null);
var prefixManagerId = _prefixManager.Write(consolidatePageStore, txnId, null);
var resourceIndexId = _resourceIndex.Write(consolidatePageStore, txnId, null);
var subjectRelatedResourceIndexId = _subjectRelatedResourceIndex.Write(consolidatePageStore, txnId, null);
var objectRelatedResourceIndexId = _objectRelatedResourceIndex.Write(consolidatePageStore, txnId, null);
var buff = CreateStoreHeader(graphIndexId, prefixManagerId, resourceIndexId, subjectRelatedResourceIndexId,
objectRelatedResourceIndexId);
var storePage = consolidatePageStore.Create(txnId);
storePage.SetData(buff);
storePage.SetData(buff, 0, 128);
consolidatePageStore.Commit(txnId, null);
// Close the stores to allow the rename to happen
Close();
consolidatePageStore.Close();
storeManager.ActivateConsolidationStore(DirectoryPath);
storeManager.GetMasterFile(DirectoryPath).AppendCommitPoint(
new CommitPoint(storePage.Id, txnId, DateTime.UtcNow, jobId), true);
}
public void TestCreateStore()
{
var storeManager = StoreManagerFactory.GetStoreManager();
var store = storeManager.CreateStore("TestCreateStore");
store.Dispose();
}
private ObjectLocationContainer GetContainerForObjectId(ulong objectId)
{
var containerNumber = (int)(objectId / ContainerSize);
// see if its loaded
if (_objectLocationContainers.ContainsKey(containerNumber))
{
return(_objectLocationContainers[containerNumber]);
}
// see if there is a offset and load the container
if (_containerOffsets.Count > containerNumber)
{
var offset = _containerOffsets[(int)containerNumber];
var storeManager = StoreManagerFactory.GetStoreManager() as IStoreManager2;
if (storeManager == null)
{
throw new Exception("Invalid store manager instance returned by store manager factory");
}
var container = storeManager.ReadObject <ObjectLocationContainer>(_storeFileName, offset);
_objectLocationContainers.Add(containerNumber, container);
return(container);
}
else
{
// else we create a new one
var container = new ObjectLocationContainer();
_objectLocationContainers.Add(containerNumber, container);
_containerOffsets.Add(0);
return(container);
}
}
/// <summary>
/// Starts the crawl of the store contained in the specified directory
/// </summary>
/// <param name="storePath">The full path to the directory that contains the store to be crawled</param>
public void Run(string storePath)
{
var dataFile = new FileInfo(Path.Combine(storePath, AbstractStoreManager.DataFileName));
var masterFile = new FileInfo(Path.Combine(storePath, AbstractStoreManager.MasterFileName));
if (!dataFile.Exists)
{
throw new FileNotFoundException("Cannot find data file", dataFile.FullName);
}
string storeLocation;
ulong nextObjectId, resourceIdIndexObjectId, graphUriToIdObjectId;
PredicateIndexResourceToObjectIdIndex propertyTypeSubjectIndex;
PredicateIndexResourceToObjectIdIndex propertyTypeObjectIndex;
var sm = StoreManagerFactory.GetStoreManager() as AbstractStoreManager;
var offset = sm.GetLatestStorePositionFromMasterFile(masterFile.FullName);
// We need to introspect the datastream directly first because Store does not currently surface direct access to index object ids
using (
var dataStream =
new BinaryReader(new FileStream(dataFile.FullName, FileMode.Open, FileAccess.Read,
FileShare.ReadWrite)))
{
dataStream.BaseStream.Seek((long)offset, SeekOrigin.Begin);
SerializationUtils.ReadVarint(dataStream);
var storeLocationSize = (int)SerializationUtils.ReadVarint(dataStream);
var locationBytes = dataStream.ReadBytes(storeLocationSize);
storeLocation = Encoding.UTF8.GetString(locationBytes, 0, storeLocationSize);
nextObjectId = SerializationUtils.ReadVarint(dataStream);
resourceIdIndexObjectId = SerializationUtils.ReadVarint(dataStream);
graphUriToIdObjectId = SerializationUtils.ReadVarint(dataStream);
_objectLocationManager = new ObjectLocationManager();
_objectLocationManager.Read(dataStream);
propertyTypeObjectIndex = new PredicateIndexResourceToObjectIdIndex();
propertyTypeObjectIndex.Read(dataStream);
propertyTypeSubjectIndex = new PredicateIndexResourceToObjectIdIndex();
propertyTypeSubjectIndex.Read(dataStream);
}
_store = sm.OpenStore(storePath, true) as Store;
var lastCommit = _store.GetCommitPoints().First();
foreach (var a in _analyzers)
{
a.OnStoreStart(_store.ObjectId, storeLocation, nextObjectId, lastCommit.CommitTime);
}
CrawlBTree <Bucket>(resourceIdIndexObjectId, "Resource String to Resource ID Index");
CrawlBTree <Bucket>(graphUriToIdObjectId, "Graph URI to Resource ID Index");
CrawlPredicateIndex(propertyTypeSubjectIndex, "Property Type Subject Index");
CrawlPredicateIndex(propertyTypeObjectIndex, "Property Type Object Index");
foreach (var a in _analyzers)
{
a.OnStoreEnd(_store.ObjectId);
}
}
/// <summary>
/// Makes the provided commit point the most recent one.
/// </summary>
/// <param name="commitPoint">The commitpoint to make the most recent.</param>
public void RevertToCommitPoint(CommitPoint commitPoint)
{
var storeManager = StoreManagerFactory.GetStoreManager();
var masterFile = storeManager.GetMasterFile(DirectoryPath);
if (masterFile.PersistenceType == PersistenceType.Rewrite)
{
throw new BrightstarClientException("Revert is not supported by a store using the binary page persistence type.");
}
masterFile.AppendCommitPoint(commitPoint);
}
/// <summary>
/// Creates a new server core
/// </summary>
/// <param name="baseLocation">Path to the stores directory</param>
/// <param name="storeName">Name of store</param>
public StoreWorker(string baseLocation, string storeName)
{
_storeName = storeName;
_storeLocation = Path.Combine(baseLocation, storeName);
Logging.LogInfo("StoreWorker created with location {0}", _storeLocation);
_jobs = new ConcurrentQueue <Job>();
// _jobStatus = new ConcurrentDictionary<string, JobStatus>();
_jobExecutionStatus = new ConcurrentDictionary <string, JobExecutionStatus>();
_storeManager = StoreManagerFactory.GetStoreManager();
_transactionLog = _storeManager.GetTransactionLog(_storeLocation);
}
public ServerCore(string baseLocation, ICache queryCache, PersistenceType persistenceType)
{
Logging.LogInfo("ServerCore Initialised {0}", baseLocation);
_baseLocation = baseLocation;
_stores = new Dictionary <string, StoreWorker>();
var configuration = new StoreConfiguration {
PersistenceType = persistenceType
};
_storeManager = StoreManagerFactory.GetStoreManager(configuration);
_queryCache = queryCache;
}
public ServerCore(string baseLocation, ICache queryCache, PersistenceType persistenceType, bool enableTransactionLoggingOnNewStores)
{
Logging.LogInfo("ServerCore Initialised {0}", baseLocation);
_baseLocation = baseLocation;
_stores = new Dictionary <string, StoreWorker>();
var configuration = StoreConfiguration.DefaultStoreConfiguration.Clone() as StoreConfiguration;
configuration.PersistenceType = persistenceType;
_storeManager = StoreManagerFactory.GetStoreManager(configuration);
_queryCache = queryCache;
_enableTransactionLogging = enableTransactionLoggingOnNewStores;
}
public void Commit(Guid jobId, BrightstarProfiler profiler = null)
{
using (profiler.Step("Store Commit"))
{
ulong storePageId = Save(profiler);
var storeManager = StoreManagerFactory.GetStoreManager();
var mf = storeManager.GetMasterFile(DirectoryPath);
mf.AppendCommitPoint(new CommitPoint(storePageId, _currentTxnId + 1, DateTime.UtcNow, jobId));
_currentTxnId++;
}
}
/// <summary>
/// Creates a new server core
/// </summary>
/// <param name="baseLocation">Path to the stores directory</param>
/// <param name="storeName">Name of store</param>
public StoreWorker(string baseLocation, string storeName)
{
_baseLocation = baseLocation;
_storeName = storeName;
_storeLocation = Path.Combine(baseLocation, storeName);
Logging.LogInfo("StoreWorker created with location {0}", _storeLocation);
_jobs = new ConcurrentQueue <Job>();
_jobExecutionStatus = new ConcurrentDictionary <string, JobExecutionStatus>();
_storeManager = StoreManagerFactory.GetStoreManager();
_transactionLog = _storeManager.GetTransactionLog(_storeLocation);
_storeStatisticsLog = _storeManager.GetStatisticsLog(_storeLocation);
_statsMonitor = new StatsMonitor();
InitializeStatsMonitor();
_shutdownCompleted = new ManualResetEvent(false);
}
private void CheckTriples(string storeName, int startId, int endId)
{
var store = StoreManagerFactory.GetStoreManager().OpenStore("C:\\brightstar\\" + storeName, true);
List <int> missingIds = new List <int>();
for (int i = startId; i < endId; i++)
{
var matchCount = store.Match("http://www.example.org/resource/" + i, null, null, graphs: null).Count();
if (matchCount == 0)
{
missingIds.Add(i);
}
}
if (missingIds.Count > 0)
{
Assert.Fail("Could match the following IDs: " + String.Join(", ", missingIds));
}
}
public OptimiserTests()
{
_storeManager = StoreManagerFactory.GetStoreManager();
_docTagStore = InitializeDocTagStore();
}
public void SetUp()
{
_storeManager = StoreManagerFactory.GetStoreManager();
}
public ManifestSyntax()
{
_storeManager = StoreManagerFactory.GetStoreManager();
}
/// <summary>
/// Returns a list of commit points with the most recent returned first
/// </summary>
/// <returns>Store commit points, most recent first.</returns>
public IEnumerable <CommitPoint> GetCommitPoints()
{
var storeManager = StoreManagerFactory.GetStoreManager();
return(storeManager.GetMasterFile(DirectoryPath).GetCommitPoints());
}
public BrightstarQueryProcessorTests()
{
_storeManager = StoreManagerFactory.GetStoreManager();
_docTagStore = InitializeDocTagStore();
}
public SparqlTest()
{
_storeManager = StoreManagerFactory.GetStoreManager();
}
