public ulong AppendCommitPoint(CommitPoint newCommitPoint, bool overwrite = false)
{
if (overwrite)
{
Logging.LogDebug("AppendCommitPoint: Overwrite requested. Deleting existing master file at '{0}'",
_masterFilePath);
_persistenceManager.DeleteFile(_masterFilePath);
_commitPoints.Clear();
}
if (!_persistenceManager.FileExists(_masterFilePath))
{
Logging.LogDebug("AppendCommitPoint: Master file not found at '{0}'. Creating new master file.",
_masterFilePath);
using (var stream = _persistenceManager.GetOutputStream(_masterFilePath, FileMode.Create))
{
Save(stream);
}
}
using (var stream = _persistenceManager.GetOutputStream(_masterFilePath, FileMode.Open))
{
stream.Seek(0, SeekOrigin.End);
newCommitPoint.Save(stream);
_commitPoints.TryAdd(stream.Length - CommitPoint.RecordSize, newCommitPoint);
return(newCommitPoint.CommitNumber);
}
}
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Commit"))
{
var pagesToWrite = _newPages.Where(rf => rf.IsAlive).Select(rf => rf.Target as IPage).Where(pg => pg != null);
if (_backgroundPageWriter != null)
{
foreach (var page in pagesToWrite)
{
_backgroundPageWriter.QueueWrite(page, commitId);
}
_backgroundPageWriter.Flush();
RestartBackgroundWriter();
}
else
{
using (var outputStream = _peristenceManager.GetOutputStream(_path, FileMode.Open))
{
foreach (var page in pagesToWrite)
{
page.Write(outputStream, commitId);
}
}
}
_newPages.Clear();
_newPageOffset = _nextPageId;
PageCache.Instance.Clear(_path);
}
}
public void AppendStatistics(StoreStatistics statistics)
{
using (
var headerWriter =
new BinaryWriter(_persistenceManager.GetOutputStream(GetStatisticsHeaderFile(), FileMode.Append)))
{
using (
var recordWriter =
new StreamWriter(_persistenceManager.GetOutputStream(GetStatisticsLogFile(),
FileMode.Append)))
{
var recordStart = recordWriter.BaseStream.Position;
var record = new StoreStatisticsRecord(statistics.CommitNumber, statistics.TripleCount,
statistics.PredicateTripleCounts);
var header = new StoreStatisticsHeaderRecord(statistics.CommitNumber, statistics.CommitTime,
recordStart);
record.Save(recordWriter);
header.Save(headerWriter);
recordWriter.Flush();
headerWriter.Flush();
recordWriter.Close();
headerWriter.Close();
}
}
}
/// <summary>
/// Logs the transaction data to the file and remembers the start position.
/// </summary>
/// <param name="itemToLog"></param>
public void LogStartTransaction(ILoggable itemToLog)
{
// get the start position
_currentTransactionDataStartPosition = (ulong)_persistenceManager.GetFileLength(GetTransactionLogFile());
// now write the transaction data
using (var stream = _persistenceManager.GetOutputStream(GetTransactionLogFile(), FileMode.Append))
{
itemToLog.LogTransactionDataToStream(stream);
}
}
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
if (CanWrite)
{
if (_backgroundPageWriter != null)
{
foreach (var pageId in _modifiedPages.Keys)
{
var page = PageCache.Instance.Lookup(_partitionId, pageId) as BinaryFilePage;
if (page != null && page.IsDirty)
{
_backgroundPageWriter.QueueWrite(page, commitId);
}
}
_backgroundPageWriter.Flush();
lock (_restartLock)
{
_backgroundPageWriter.Shutdown();
_backgroundPageWriter.Dispose();
PageCache.Instance.Clear(_partitionId);
UpdatePartitionId();
_readTxnId = _writeTxnId;
_writeTxnId++;
_backgroundPageWriter =
new BackgroundPageWriter(_persistenceManager.GetOutputStream(_filePath, FileMode.Open));
}
}
else
{
using (var outputStream = _persistenceManager.GetOutputStream(_filePath, FileMode.Open))
{
foreach (var pageId in _modifiedPages.Keys.OrderBy(x => x))
{
var page = PageCache.Instance.Lookup(_partitionId, pageId) as BinaryFilePage;
if (page != null && page.IsDirty)
{
page.Write(outputStream, commitId);
}
}
outputStream.Flush();
}
PageCache.Instance.Clear(_partitionId);
UpdatePartitionId();
_readTxnId = _writeTxnId;
_writeTxnId++;
}
}
else
{
throw new InvalidOperationException("Attempt to Commit on a read-only store instance");
}
}
private void CopyTestDataToImportFolder(string testDataFileName, string targetFileName = null)
{
#if PORTABLE
using (var srcStream = _persistenceManager.GetInputStream(Configuration.DataLocation + testDataFileName))
{
var targetDir = Path.Combine(Configuration.StoreLocation, "import");
var targetPath = Path.Combine(targetDir, (targetFileName ?? testDataFileName));
if (!_persistenceManager.DirectoryExists(targetDir))
{
_persistenceManager.CreateDirectory(targetDir);
}
if (_persistenceManager.FileExists(targetPath))
{
_persistenceManager.DeleteFile(targetPath);
}
using (var targetStream = _persistenceManager.GetOutputStream(targetPath, FileMode.CreateNew))
{
srcStream.CopyTo(targetStream);
}
}
#else
var importFile = new FileInfo(Path.Combine(Configuration.DataLocation, testDataFileName));
var targetDir = new DirectoryInfo(Path.Combine(Configuration.StoreLocation, "import"));
if (!targetDir.Exists)
{
targetDir.Create();
}
importFile.CopyTo(Path.Combine(targetDir.FullName, targetFileName ?? testDataFileName), true);
#endif
}
public AppendOnlyFilePageStore(IPersistenceManager persistenceManager, string filePath, int pageSize, bool readOnly, bool disableBackgroundWrites)
{
_peristenceManager = persistenceManager;
_path = filePath;
if ((_pageSize % 4096) != 0)
{
throw new ArgumentException("Page size must be a multiple of 4096 bytes");
}
_pageSize = pageSize;
_bitShift = (int)Math.Log(_pageSize, 2.0);
if (!_peristenceManager.FileExists(filePath) && !readOnly)
{
// Create an empty file that we can write to later
_peristenceManager.CreateFile(filePath);
}
_stream = _peristenceManager.GetInputStream(_path);
_nextPageId = ((ulong)_stream.Length >> _bitShift) + 1;
if (!readOnly)
{
_newPages = new List <IPage>(512);
_newPageOffset = _nextPageId;
}
_pageSize = pageSize;
_readonly = readOnly;
if (!readOnly && !disableBackgroundWrites)
{
_backgroundPageWriter =
new BackgroundPageWriter(persistenceManager.GetOutputStream(filePath, FileMode.Open));
}
}
/// <summary>
/// Commits all changed and new pages to the page store
/// </summary>
/// <param name="commitId">The transaction identifier for the commit</param>
/// <param name="profiler"></param>
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Commit"))
{
using (var outputStream = _persistenceManager.GetOutputStream(_filePath, FileMode.Open))
{
try
{
foreach (var entry in _modifiedPages.OrderBy(e => e.Key))
{
entry.Value.Write(outputStream, commitId);
lock (_pageCacheLock)
{
PageCache.Instance.InsertOrUpdate(_filePath, entry.Value);
}
}
_modifiedPages.Clear();
}
catch (Exception)
{
_modifiedPages.Clear();
throw;
}
}
CurrentTransactionId = commitId;
}
}
public AppendOnlyFilePageStore(IPersistenceManager persistenceManager, string filePath, int pageSize, bool readOnly, bool disableBackgroundWrites)
{
_peristenceManager = persistenceManager;
_path = filePath;
if ((_pageSize % 4096) != 0)
{
throw new ArgumentException("Page size must be a multiple of 4096 bytes");
}
_pageSize = pageSize;
_bitShift = (int)Math.Log(_pageSize, 2.0);
if (!_peristenceManager.FileExists(filePath) && !readOnly)
{
// Create an empty file that we can write to later
_peristenceManager.CreateFile(filePath);
}
_stream = _peristenceManager.GetInputStream(_path);
_nextPageId = ((ulong)_stream.Length >> _bitShift) + 1;
if (!readOnly)
{
_newPages = new List<IPage>(512);
_newPageOffset = _nextPageId;
}
_pageSize = pageSize;
_readonly = readOnly;
if (!readOnly && !disableBackgroundWrites)
{
_backgroundPageWriter =
new BackgroundPageWriter(persistenceManager.GetOutputStream(filePath, FileMode.Open));
}
}
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Commit"))
{
if (_backgroundPageWriter != null)
{
foreach (var p in _newPages)
{
_backgroundPageWriter.QueueWrite(p, commitId);
}
_backgroundPageWriter.Flush();
RestartBackgroundWriter();
foreach (var p in _newPages)
{
PageCache.Instance.InsertOrUpdate(_path, p);
}
}
else
{
using (var outputStream = _peristenceManager.GetOutputStream(_path, FileMode.Open))
{
foreach (var p in _newPages)
{
p.Write(outputStream, commitId);
PageCache.Instance.InsertOrUpdate(_path, p);
}
}
}
_newPages.Clear();
_newPageOffset = _nextPageId;
}
/*
* using (var writeStream = _peristenceManager.GetOutputStream(_path, FileMode.Open))
* {
* writeStream.Seek((long) ((_newPageOffset - 1)*(ulong) _pageSize), SeekOrigin.Begin);
* foreach (var p in _newPages)
* {
* writeStream.Write(p.Data, 0, _pageSize);
* }
* writeStream.Flush();
* _newPages.Clear();
* _newPageOffset = _nextPageId;
* }
*/
}
public void TestOpenForWriting()
{
if (_pm.FileExists("writeme.txt"))
{
_pm.DeleteFile("writeme.txt");
}
Assert.IsFalse(_pm.FileExists("writeme.txt"));
using (var writer = new StreamWriter(_pm.GetOutputStream("writeme.txt", FileMode.CreateNew)))
{
writer.Write("Hello world");
}
Assert.IsTrue(_pm.FileExists("writeme.txt"));
Assert.AreEqual(11, _pm.GetFileLength("writeme.txt"));
// Test using Open mode
using (var writer = new StreamWriter(_pm.GetOutputStream("writeme.txt", FileMode.Open)))
{
writer.BaseStream.Seek(0, SeekOrigin.End);
writer.Write(" and goodbye");
}
Assert.AreEqual(23, _pm.GetFileLength("writeme.txt"));
// Test using Truncate mode
using (var writer = new StreamWriter(_pm.GetOutputStream("writeme.txt", FileMode.Truncate)))
{
writer.Write("Hello world");
}
Assert.AreEqual(11, _pm.GetFileLength("writeme.txt"));
// Test using Append mode
using (var writer = new StreamWriter(_pm.GetOutputStream("writeme.txt", FileMode.Append)))
{
writer.Write(" and goodbye");
}
Assert.AreEqual(23, _pm.GetFileLength("writeme.txt"));
}
public void Commit(ulong commitId, BrightstarProfiler profiler)
{
using (profiler.Step("PageStore.Commit"))
{
var livePages = new List <IPage>();
if (_backgroundPageWriter != null)
{
foreach (var p in _newPages)
{
if (p.IsAlive)
{
_backgroundPageWriter.QueueWrite(p.Target as IPage, commitId);
livePages.Add(p.Target as IPage);
}
}
_backgroundPageWriter.Flush();
RestartBackgroundWriter();
PageCache.Instance.Clear(_path);
}
else
{
using (var outputStream = _peristenceManager.GetOutputStream(_path, FileMode.Open))
{
foreach (var p in _newPages)
{
if (p.IsAlive)
{
(p.Target as IPage).Write(outputStream, commitId);
}
}
}
}
_newPages.Clear();
_newPageOffset = _nextPageId;
}
}
public static MasterFile Create(IPersistenceManager persistenceManager, string directoryPath,
StoreConfiguration storeConfiguration, Guid storeSetId)
{
var masterFilePath = Path.Combine(directoryPath, MasterFileName);
if (persistenceManager.FileExists(masterFilePath))
{
throw new StoreManagerException(directoryPath, "Master file already exists");
}
persistenceManager.CreateFile(masterFilePath);
using (var stream = persistenceManager.GetOutputStream(masterFilePath, FileMode.Open))
{
var newMaster = new MasterFile(persistenceManager, directoryPath, masterFilePath, storeConfiguration,
storeSetId);
newMaster.Save(stream);
return newMaster;
}
}
public static MasterFile Create(IPersistenceManager persistenceManager, string directoryPath,
StoreConfiguration storeConfiguration, Guid storeSetId)
{
var masterFilePath = Path.Combine(directoryPath, MasterFileName);
if (persistenceManager.FileExists(masterFilePath))
{
throw new StoreManagerException(directoryPath, "Master file already exists");
}
persistenceManager.CreateFile(masterFilePath);
using (var stream = persistenceManager.GetOutputStream(masterFilePath, FileMode.Open))
{
var newMaster = new MasterFile(persistenceManager, directoryPath, masterFilePath, storeConfiguration,
storeSetId);
newMaster.Save(stream);
return(newMaster);
}
}
public BinaryFilePageStore(IPersistenceManager persistenceManager, string filePath, int pageSize, bool readOnly,
ulong transactionId, ulong nextTransactionId)
{
_persistenceManager = persistenceManager;
_nominalPageSize = pageSize;
_filePath = filePath;
_readTxnId = transactionId;
_writeTxnId = nextTransactionId;
PageSize = _nominalPageSize - 8;
CanWrite = !readOnly;
OpenInputStream();
_nextPageId = (ulong)_inputStream.Length / ((uint)_nominalPageSize * 2) + 1;
if (CanWrite)
{
_backgroundPageWriter =
new BackgroundPageWriter(_persistenceManager.GetOutputStream(_filePath, FileMode.Open));
PageCache.Instance.BeforeEvict += BeforePageCacheEvict;
}
_modifiedPages = new ConcurrentDictionary <ulong, bool>();
UpdatePartitionId();
}
/// <summary>
/// Background write of log messages to log file.
/// </summary>
static void DoWriteLog()
{
while (!_stopping)
{
if (!LogQueue.IsEmpty)
{
if (!String.IsNullOrEmpty(_logFileName))
{
using (var logStream = _persistenceManager.GetOutputStream(_logFileName, FileMode.Append))
{
using (var logWriter = new StreamWriter(logStream))
{
LogItem logItem;
while (LogQueue.TryDequeue(out logItem))
{
logItem.WriteTo(logWriter);
}
logStream.Flush();
logWriter.Close();
}
}
}
else
{
// Not currently logging to a file - so just dequeue all the messages
LogItem logItem;
while (LogQueue.TryDequeue(out logItem))
{
// No-op
}
}
}
Wait.Reset();
Wait.WaitOne(1000);
}
}
public void ConsolidateStore(Store store, string storeLocation, Guid jobId)
{
// delete consolidate file if for some reason it was still there
if (_persistenceManager.FileExists(store.StoreConsolidateFile))
{
_persistenceManager.DeleteFile(store.StoreConsolidateFile);
}
// var inputStream = _persistenceManager.GetInputStream(store.StoreDataFile);
var inputStream = store.InputDataStream;
var outputStream = _persistenceManager.GetOutputStream(store.StoreConsolidateFile, FileMode.CreateNew);
ulong offset = 0;
try
{
using (var writer = new BinaryWriter(outputStream))
{
var objectLocationManager = store.ObjectLocationManager;
foreach (var container in objectLocationManager.Containers)
{
foreach (var objLoc in container.ObjectOffsets)
{
// load object
var objType = GetObjectType(objLoc.Type);
if (objType.Equals(typeof(Store)))
{
// dont write the store.
continue;
}
var obj = ReadObject(inputStream, objLoc.Offset, GetObjectType(objLoc.Type));
// save object and update location manager
var bytes = obj.Save(writer, offset);
// manage offsets
objectLocationManager.SetObjectOffset(obj.ObjectId, offset,
PersistantTypeIdentifiers[obj.GetType()], 1);
offset += (ulong)bytes;
}
}
// write the store
objectLocationManager.SetObjectOffset(store.ObjectId, offset,
PersistantTypeIdentifiers[store.GetType()], 1);
store.Save(writer, offset);
// delete store
inputStream.Close();
inputStream.Dispose();
_persistenceManager.DeleteFile(store.StoreDataFile);
}
// rename to new
_persistenceManager.RenameFile(store.StoreConsolidateFile, store.StoreDataFile);
// update the masterfile
AppendCommitPoint(storeLocation, new CommitPoint(offset, 0ul, DateTime.UtcNow, jobId), true);
}
catch (Exception ex)
{
Logging.LogError(new BrightstarEventId(), "Unable to consolidate store. Error was " + ex.Message);
throw;
}
finally
{
_persistenceManager.DeleteFile(store.StoreConsolidateFile);
}
}
private void EnsureOutputStream()
{
_outputStream = _persistenceManager.GetOutputStream(_filePath, FileMode.Open);
}
public BinaryFilePageStore(IPersistenceManager persistenceManager, string filePath, int pageSize, bool readOnly,
ulong transactionId, ulong nextTransactionId)
{
_persistenceManager = persistenceManager;
_nominalPageSize = pageSize;
_filePath = filePath;
_readTxnId = transactionId;
_writeTxnId = nextTransactionId;
PageSize = _nominalPageSize - 8;
CanWrite = !readOnly;
OpenInputStream();
_nextPageId = (ulong) _inputStream.Length/((uint) _nominalPageSize*2) + 1;
if (CanWrite)
{
_backgroundPageWriter =
new BackgroundPageWriter(_persistenceManager.GetOutputStream(_filePath, FileMode.Open));
PageCache.Instance.BeforeEvict += BeforePageCacheEvict;
}
_modifiedPages = new ConcurrentDictionary<ulong, bool>();
UpdatePartitionId();
}