public void TestResourceTableInsert()
{
const string testString = "this is a test string";
ulong pageId;
byte segmentId;
using (var pageStore = TestUtils.CreateEmptyPageStore("ResourceTableInsert"))
{
var resourceTable = new ResourceTable(pageStore);
resourceTable.Insert(0, testString, out pageId, out segmentId, null);
Assert.AreEqual((ulong)1, pageId);
Assert.AreEqual((byte)0, segmentId);
// Test we can retrieve it from the currently open page store
var resource = resourceTable.GetResource(pageId, segmentId, null);
Assert.AreEqual(testString, resource);
pageStore.Commit(0, null);
}
using (var pageStore = new AppendOnlyFilePageStore(PersistenceManager, "ResourceTableInsert", 4096, true, false))
{
var resourceTable = new ResourceTable(pageStore);
var resource = resourceTable.GetResource(pageId, segmentId, null);
Assert.AreEqual(testString, resource);
}
}
public void TestBatchInsert()
{
const int target = 40000000;
const int batchSize = 50000;
const int batchCount = target/batchSize;
// Tests insert of 40 million unique keys in batches of 50,000
// Reports time for each batch to console.
#if SILVERLIGHT
var persistenceManager = new IsolatedStoragePersistanceManager();
#else
var persistenceManager = new FilePersistenceManager();
#endif
// Create a test batch
if (!File.Exists("C:\\brightstar\\testdata.dat"))
{
MakeTestData();
}
var testList = ReadTestData("c:\\brightstar\\testdata.dat");
// Create empty store
if (File.Exists("40m_batch.data")) File.Delete("40m_batch.data");
var pageStore = new AppendOnlyFilePageStore(persistenceManager, "40m_batch.data", 4096, false, false);
var tree = new BPlusTree(pageStore);
ulong lastRoot = tree.RootId;
tree.Save(0, null);
pageStore.Commit(0ul, null);
byte[] testBuffer = Encoding.UTF8.GetBytes("Test Buffer");
var batchTimer = Stopwatch.StartNew();
int insertedCount = 0;
var txnId = 1ul;
for (int i = 0; i < batchCount; i++)
{
tree = new BPlusTree(pageStore, lastRoot);
foreach (var item in testList)
{
var insertKey = (ulong) ((item*batchCount) + i);
try
{
tree.Insert(txnId, insertKey, testBuffer);
insertedCount++;
}
catch (Exception e)
{
Assert.Fail("Failed to add key {0}. Cause: {1}", insertKey, e);
}
}
long beforeSave = batchTimer.ElapsedMilliseconds;
tree.Save(txnId, null);
lastRoot = tree.RootId;
pageStore.Commit(txnId, null);
Console.WriteLine("{0},{1},{2}", insertedCount, beforeSave, batchTimer.ElapsedMilliseconds);
txnId++;
}
}
public IStore CreateStore(string storeLocation, PersistenceType storePersistenceType, bool readOnly, bool withTransactionLogging)
{
Logging.LogInfo("Create Store {0} with persistence type {1}", storeLocation, storePersistenceType);
if (_persistenceManager.DirectoryExists(storeLocation))
{
throw new StoreManagerException(storeLocation, "Store already exists");
}
_persistenceManager.CreateDirectory(storeLocation);
var dataFilePath = Path.Combine(storeLocation, DataFileName);
_persistenceManager.CreateFile(dataFilePath);
var resourceFilePath = Path.Combine(storeLocation, ResourceFileName);
_persistenceManager.CreateFile(resourceFilePath);
var targetStoreConfiguration = _storeConfiguration.Clone() as StoreConfiguration;
targetStoreConfiguration.PersistenceType = storePersistenceType;
MasterFile.Create(_persistenceManager, storeLocation, targetStoreConfiguration, Guid.NewGuid());
IPageStore dataPageStore = null;
switch (storePersistenceType)
{
case PersistenceType.AppendOnly:
dataPageStore = new AppendOnlyFilePageStore(_persistenceManager, dataFilePath, PageSize, false, _storeConfiguration.DisableBackgroundWrites);
break;
case PersistenceType.Rewrite:
dataPageStore = new BinaryFilePageStore(_persistenceManager, dataFilePath, PageSize, false, 0, 1, _storeConfiguration.DisableBackgroundWrites);
break;
}
IPageStore resourcePageStore = new AppendOnlyFilePageStore(_persistenceManager, resourceFilePath, PageSize, false, _storeConfiguration.DisableBackgroundWrites);
var resourceTable = new ResourceTable(resourcePageStore);
using (var store = new Store(storeLocation, dataPageStore, resourceTable))
{
store.Commit(Guid.Empty);
store.Close();
}
if (withTransactionLogging)
{
_persistenceManager.CreateFile(Path.Combine(storeLocation, "transactionheaders.bs"));
_persistenceManager.CreateFile(Path.Combine(storeLocation, "transactions.bs"));
}
Logging.LogInfo("Store created at {0}", storeLocation);
return OpenStore(storeLocation, readOnly);
}
public void TestAddFirstPageToEmptyPageStore()
{
//if (File.Exists(PageFilePath)) File.Delete(PageFilePath);
if (PersistenceManager.FileExists(PageFilePath)) PersistenceManager.DeleteFile(PageFilePath);
using (var readWritePageStore = new AppendOnlyFilePageStore(PersistenceManager, PageFilePath, 8192, false, false))
{
var createdPage = readWritePageStore.Create(1);
Assert.AreEqual(1ul, createdPage.Id);
createdPage.SetData(TestBuffer);
ValidateBuffer(readWritePageStore.Retrieve(createdPage.Id, null).Data);
readWritePageStore.Commit(1ul, null);
}
using (var readPageStore = new AppendOnlyFilePageStore(PersistenceManager, PageFilePath, 8192, true, false))
{
var page = readPageStore.Retrieve(1ul, null);
ValidateBuffer(page.Data);
}
}
public void TestCreateAppendOnlyPageStore()
{
if (PersistenceManager.FileExists(PageFilePath)) PersistenceManager.DeleteFile(PageFilePath);
using (var readwritePageStore =
new AppendOnlyFilePageStore(PersistenceManager, PageFilePath,8192, false, false))
{
Assert.AreEqual(8192, readwritePageStore.PageSize);
Assert.IsTrue(readwritePageStore.CanRead);
Assert.IsTrue(readwritePageStore.CanWrite);
using (var readonlyPageStore =
new AppendOnlyFilePageStore(PersistenceManager, PageFilePath, 8192, true, false))
{
Assert.AreEqual(8192, readonlyPageStore.PageSize);
Assert.IsTrue(readonlyPageStore.CanRead);
Assert.IsFalse(readonlyPageStore.CanWrite);
}
}
Assert.IsTrue(PersistenceManager.FileExists(PageFilePath));
Assert.AreEqual(0, PersistenceManager.GetFileLength(PageFilePath));
}
public void TestInsertLongResources()
{
var longResourceA = new string('A', 3070);
var longResourceB = new string('B', 3070);
var longResourceC = new string('C', 3070);
ulong pageA, pageB, pageC;
byte segA, segB, segC;
using (var pageStore = TestUtils.CreateEmptyPageStore("ResourceTableInsertLong"))
{
var resourceTable = new ResourceTable(pageStore);
resourceTable.Insert(0, longResourceA, out pageA, out segA, null);
resourceTable.Insert(0, longResourceB, out pageB, out segB, null);
resourceTable.Insert(0, longResourceC, out pageC, out segC, null);
// Test we can retrieve the long resources from the currently open page store
var resource = resourceTable.GetResource(pageA, segA, null);
Assert.AreEqual(longResourceA, resource);
resource = resourceTable.GetResource(pageB, segB, null);
Assert.AreEqual(longResourceB, resource);
resource = resourceTable.GetResource(pageC, segC, null);
Assert.AreEqual(longResourceC, resource);
pageStore.Commit(0, null);
}
using (var pageStore = new AppendOnlyFilePageStore(PersistenceManager, "ResourceTableInsertLong", 4096, true, false))
{
var resourceTable = new ResourceTable(pageStore);
var resource = resourceTable.GetResource(pageA, segA, null);
Assert.AreEqual(longResourceA, resource);
resource = resourceTable.GetResource(pageB, segB, null);
Assert.AreEqual(longResourceB, resource);
resource = resourceTable.GetResource(pageC, segC, null);
Assert.AreEqual(longResourceC, resource);
}
}
public IStore OpenStore(string storeLocation, bool readOnly)
{
Logging.LogInfo("Open Store {0}", storeLocation);
var masterFile = MasterFile.Open(_persistenceManager, storeLocation);
var latestCommitPoint = masterFile.GetLatestCommitPoint();
var dataFilePath = Path.Combine(storeLocation, DataFileName);
var resourceFilePath = Path.Combine(storeLocation, ResourceFileName);
if (_persistenceManager.FileExists(dataFilePath))
{
IPageStore dataPageStore = null;
switch (masterFile.PersistenceType)
{
case PersistenceType.AppendOnly:
dataPageStore = new AppendOnlyFilePageStore(_persistenceManager, dataFilePath, PageSize, readOnly, _storeConfiguration.DisableBackgroundWrites);
break;
case PersistenceType.Rewrite:
dataPageStore = new BinaryFilePageStore(_persistenceManager, dataFilePath, PageSize, readOnly, latestCommitPoint.CommitNumber);
break;
}
var resourcePageStore = new AppendOnlyFilePageStore(_persistenceManager, resourceFilePath, PageSize, readOnly, _storeConfiguration.DisableBackgroundWrites);
var resourceTable = new ResourceTable(resourcePageStore);
var store = new Store(storeLocation, dataPageStore, resourceTable, latestCommitPoint.LocationOffset, null);
Logging.LogInfo("Store {0} opened successfully", storeLocation);
return store;
}
throw new StoreManagerException(storeLocation, "Data file not found");
}
/// <summary>
/// Opens the store at the specified commit point (always read-only)
/// </summary>
/// <param name="storeLocation"></param>
/// <param name="commitPointId"></param>
/// <returns></returns>
public IStore OpenStore(string storeLocation, ulong commitPointId)
{
if (_storeConfiguration.PersistenceType == PersistenceType.Rewrite)
{
throw new InvalidOperationException("Rewrite page store does not support opening at a previous commit point");
}
Logging.LogInfo("Open Store {0} at commit point {1}", storeLocation, commitPointId);
var dataFilePath = Path.Combine(storeLocation, DataFileName);
var resourceFilePath = Path.Combine(storeLocation, ResourceFileName);
var commitPoint = GetMasterFile(storeLocation).GetCommitPoints().FirstOrDefault(cp => cp.LocationOffset == commitPointId);
if (commitPoint != null)
{
if (!_persistenceManager.FileExists(resourceFilePath))
{
throw new StoreManagerException(storeLocation, "Resource file not found");
}
if (!_persistenceManager.FileExists(dataFilePath))
{
throw new StoreManagerException(storeLocation, "Data file not found");
}
var pageStore = new AppendOnlyFilePageStore(_persistenceManager, dataFilePath, PageSize, true, _storeConfiguration.DisableBackgroundWrites);
var resourceStore = new AppendOnlyFilePageStore(_persistenceManager, resourceFilePath, PageSize, true, _storeConfiguration.DisableBackgroundWrites);
var resourceTable = new ResourceTable(resourceStore);
var store = new Store(storeLocation, pageStore, resourceTable, commitPointId, null);
return store;
}
throw new StoreManagerException(storeLocation, "Commit point not found");
}
public void CreateSnapshot(string srcStoreLocation, string destStoreLocation,
PersistenceType storePersistenceType, ulong commitPointId = StoreConstants.NullUlong)
{
Logging.LogInfo("Snapshot store {0} to new store {1} with persistence type {2}", srcStoreLocation,
destStoreLocation, storePersistenceType);
if (_persistenceManager.DirectoryExists(destStoreLocation))
{
throw new StoreManagerException(destStoreLocation, "Store already exists");
}
// Open the source store for reading
using (IStore srcStore = commitPointId == StoreConstants.NullUlong
? OpenStore(srcStoreLocation, true)
: OpenStore(srcStoreLocation, commitPointId))
{
// Create the directory for the destination store
_persistenceManager.CreateDirectory(destStoreLocation);
// Create empty data file
var dataFilePath = Path.Combine(destStoreLocation, DataFileName);
_persistenceManager.CreateFile(dataFilePath);
// Create initial master file
var destStoreConfiguration = _storeConfiguration.Clone() as StoreConfiguration;
destStoreConfiguration.PersistenceType = storePersistenceType;
var destMasterFile = MasterFile.Create(_persistenceManager, destStoreLocation, destStoreConfiguration,
Guid.NewGuid());
// Copy resource files from source store
var resourceFilePath = Path.Combine(destStoreLocation, ResourceFileName);
_persistenceManager.CopyFile(Path.Combine(srcStoreLocation, ResourceFileName), resourceFilePath, true);
// Initialize data page store
IPageStore destPageStore = null;
switch (storePersistenceType)
{
case PersistenceType.AppendOnly:
destPageStore = new AppendOnlyFilePageStore(_persistenceManager, dataFilePath, PageSize, false,
_storeConfiguration.DisableBackgroundWrites);
break;
case PersistenceType.Rewrite:
destPageStore = new BinaryFilePageStore(_persistenceManager, dataFilePath, PageSize, false, 0, 1, _storeConfiguration.DisableBackgroundWrites);
break;
default:
throw new BrightstarInternalException("Unrecognized target store type: " + storePersistenceType);
}
// Copy Data
ulong destStorePageId = srcStore.CopyTo(destPageStore, 1ul);
destPageStore.Close();
destMasterFile.AppendCommitPoint(
new CommitPoint(destStorePageId, 1ul, DateTime.UtcNow, Guid.Empty), true);
}
}