public void PersistentIndexVersionsWorkSimultaneously()
{
IBlobManager blobManager = new InMemoryBlobManager();
LazinatorSerializationOptions options = new LazinatorSerializationOptions(IncludeChildrenMode.IncludeAllChildren, false, false, true, 20, 5);
BinaryTree = new LazinatorBinaryTree <WDouble>();
BinaryTree.Add(1.0);
BinaryTree.Add(2.0);
LazinatorMemory serialized0 = BinaryTree.SerializeLazinator(options);
PersistentIndex index0 = new PersistentIndex("simultest", blobManager, false);
index0.PersistLazinatorMemory(serialized0);
var loadedFrom0 = index0.GetLazinatorMemory();
BinaryTree = new LazinatorBinaryTree <WDouble>(loadedFrom0);
BinaryTree.Add(3.0);
LazinatorMemory serialized1 = BinaryTree.SerializeLazinator(options);
PersistentIndex index1 = new PersistentIndex(index0);
index1.PersistLazinatorMemory(serialized1);
// reload both indices
index0 = PersistentIndex.ReadFromBlob(blobManager, "simultest", null, 0);
index1 = PersistentIndex.ReadFromBlob(blobManager, "simultest", null, 1);
var tree0Reloaded = new LazinatorBinaryTree <WDouble>(index0.GetLazinatorMemory());
var tree1Reloaded = new LazinatorBinaryTree <WDouble>(index1.GetLazinatorMemory());
tree0Reloaded.Count().Should().Be(2);
tree1Reloaded.Count().Should().Be(3);
}
public async Task BinaryTreeTest_ReloadingFromBlobsAsync(bool useFile, bool containedInSingleBlob, bool recreateIndex, bool poolMemory)
{
IBlobManager blobManager = GetBlobManager(useFile, poolMemory);
string fullPath = GetPathForIndexAndBlobs(useFile, true);
if (fullPath == null)
{
return;
}
await MultipleRoundsOfRandomChangesAsync(10, 10, 10, async() =>
{
LazinatorMemory multipleBufferResult = BinaryTree.SerializeLazinator(new LazinatorSerializationOptions(IncludeChildrenMode.IncludeAllChildren, false, false, false, 5));
// Write to one or more blobs
PersistentIndex index = new PersistentIndex(fullPath, blobManager, containedInSingleBlob);
await index.PersistLazinatorMemoryAsync(multipleBufferResult);
if (recreateIndex)
{
index = await PersistentIndex.ReadFromBlobAsync(blobManager, fullPath, null, index.IndexVersion);
}
var revisedMemory = await index.GetLazinatorMemoryAsync();
BinaryTree = new LazinatorBinaryTree <WDouble>(revisedMemory);
});
if (poolMemory)
{
blobManager.MemoryAllocator.FreeMemory(fullPath);
}
}
private async Task SplittableEntitiesSavedHelper_Async(bool containedInSingleBlob, bool useFile, bool recreateIndex, bool poolMemory)
{
Example e = GetTypicalExample();
LazinatorMemory multipleBufferResult = await e.SerializeLazinatorAsync(new LazinatorSerializationOptions(IncludeChildrenMode.IncludeAllChildren, false, false, false, 10));
// Write to one or more blobs
IBlobManager blobManager = GetBlobManager(useFile, poolMemory);
string fullPath = GetPathForIndexAndBlobs(useFile, false);
if (fullPath == null)
{
return;
}
PersistentIndex index = new PersistentIndex(fullPath, blobManager, containedInSingleBlob);
await index.PersistLazinatorMemoryAsync(multipleBufferResult);
// Note: Index reference is first var indexReference = memoryReferenceInBlobs[0];
// Read from one or more blobs
if (recreateIndex)
{
index = await PersistentIndex.ReadFromBlobAsync(blobManager, fullPath, null, index.IndexVersion);
}
var revisedMemory = await index.GetLazinatorMemoryAsync();
Example e2 = new Example(revisedMemory);
ExampleEqual(e, e2).Should().BeTrue();
if (poolMemory)
{
blobManager.MemoryAllocator.FreeMemory(fullPath);
}
}
public void PersistentIndexTest(bool useFile, bool containedInSingleBlob, bool recreateIndex)
{
List <byte> fullSequence = new List <byte>()
{
1, 2, 3, 4, 5, 6, 7, 8, 9, 10
};
var initialBytes = new ReadOnlyBytes(new byte[3] {
1, 2, 3
});
var nextBytes = new ReadOnlyBytes(new byte[3] {
4, 5, 6
});
MemoryChunk nextBytesAsChunk = new MemoryChunk(nextBytes, new MemoryBlockLoadingInfo(1, 3), false);
var lastBytes = new ReadOnlyBytes(new byte[4] {
7, 8, 9, 10
});
MemoryChunk lastBytesAsChunk = new MemoryChunk(lastBytes, new MemoryBlockLoadingInfo(2, 4), false);
LazinatorMemory initialMemory = new LazinatorMemory(initialBytes);
LazinatorMemory memory1 = initialMemory.WithAppendedChunk(nextBytesAsChunk).WithAppendedChunk(lastBytesAsChunk);
IBlobManager blobManager = useFile ? new FileBlobManager() : new InMemoryBlobManager();
string fullPath = GetPathForIndexAndBlobs(useFile, true);
if (fullPath == null)
{
return;
}
PersistentIndex index = new PersistentIndex(fullPath, blobManager, containedInSingleBlob);
index.PersistLazinatorMemory(memory1);
if (recreateIndex)
{
index = PersistentIndex.ReadFromBlob(blobManager, fullPath, null, index.IndexVersion);
}
var memory2 = index.GetLazinatorMemory();
var memory2List = memory2.GetConsolidatedMemory().ToArray().ToList();
memory2List.SequenceEqual(fullSequence).Should().BeTrue();
BufferWriter writer = new BufferWriter(0, memory2);
writer.Write((byte)11);
writer.Write((byte)12);
writer.InsertReferenceToCompletedMemory(2, 1, 2); // 8, 9
writer.InsertReferenceToCompletedMemory(0, 0, 3); // 1, 2, 3
writer.InsertReferenceToCompletedMemory(1, 1, 1); // 5
writer.InsertReferenceToCompletedMemory(2, 0, 2); // 7, 8
writer.Write((byte)13);
var memory3 = writer.LazinatorMemory;
var memory3List = memory3.GetConsolidatedMemory().ToArray().ToList();
List <byte> expected = new List <byte>()
{
11, 12, 8, 9, 1, 2, 3, 5, 7, 8, 13
};
memory3List.SequenceEqual(expected).Should().BeTrue();
}
public void BinaryTreeTest_DiffSerialization(bool useFile, bool containedInSingleBlob, bool recreateIndex, bool neverIncludeReferenceToPreviousBuffer, bool useConsolidatedMemory)
{
List <PersistentIndex> indices = new List <PersistentIndex>();
IBlobManager blobManager = useFile ? new FileBlobManager() : new InMemoryBlobManager();
string fullPath = GetPathForIndexAndBlobs(useFile, true);
if (fullPath == null)
{
return;
}
int round = 0;
int numRounds = 2; // DEBUG 10
int numChangesFirstRound = 1; // DEBUG 10
int numChangesSubsequentRounds = 1; // DEBUG 3
MultipleRoundsOfRandomChanges(numRounds, numChangesFirstRound, numChangesSubsequentRounds, () =>
{
//Debug.WriteLine($"Round {round}");
LazinatorSerializationOptions options = neverIncludeReferenceToPreviousBuffer ? new LazinatorSerializationOptions(IncludeChildrenMode.IncludeAllChildren, false, false, true, int.MaxValue, int.MaxValue) : new LazinatorSerializationOptions(IncludeChildrenMode.IncludeAllChildren, false, false, true, 20, 5);
LazinatorMemory multipleBufferResult = BinaryTree.SerializeLazinator(options);
// DEBUG
Debug.WriteLine("Multiple buffer result:");
Debug.WriteLine(multipleBufferResult.ToStringByChunk());
Debug.WriteLine($"Consolidated{round}: " + multipleBufferResult.ToStringConsolidated());
// Write to one or more blobs
var index = (useConsolidatedMemory || indices == null || !indices.Any()) ? new PersistentIndex(fullPath, blobManager, containedInSingleBlob) : new PersistentIndex(indices.Last());
index.PersistLazinatorMemory(multipleBufferResult);
indices.Add(index);
if (recreateIndex)
{
index = PersistentIndex.ReadFromBlob(blobManager, fullPath, null, index.IndexVersion);
}
var revisedMemory = index.GetLazinatorMemory();
if (useConsolidatedMemory)
{
var consolidatedMemory = revisedMemory.GetConsolidatedMemory();
revisedMemory = new LazinatorMemory(consolidatedMemory);
if (round != numRounds - 1)
{
indices.RemoveAt(0); // with consolidated memory, we're not using diff serialization
}
}
// DEBUG
Debug.WriteLine("Revised memory result:");
Debug.WriteLine(revisedMemory.ToStringByChunk());
Debug.WriteLine($"Consolidated{round}: " + revisedMemory.ToStringConsolidated());
BinaryTree = new LazinatorBinaryTree <WDouble>(revisedMemory);
round++;
});
DeleteChunksAndIndex(useConsolidatedMemory, indices, blobManager);
}
public async Task BinaryTreeTest_DiffSerializationAsync(bool useFile, bool containedInSingleBlob, bool recreateIndex, bool neverIncludeReferenceToPreviousBuffer, bool useConsolidatedMemory)
{
List <PersistentIndex> indices = new List <PersistentIndex>();
IBlobManager blobManager = useFile ? new FileBlobManager() : new InMemoryBlobManager();
string fullPath = GetPathForIndexAndBlobs(useFile, true);
if (fullPath == null)
{
return;
}
int round = 0;
int numRounds = 10;
await MultipleRoundsOfRandomChangesAsync(numRounds, 10, 3, async() =>
{
//Debug.WriteLine($"Round {round}");
LazinatorSerializationOptions options = neverIncludeReferenceToPreviousBuffer ? new LazinatorSerializationOptions(IncludeChildrenMode.IncludeAllChildren, false, false, true, int.MaxValue, int.MaxValue) : new LazinatorSerializationOptions(IncludeChildrenMode.IncludeAllChildren, false, false, true, 20, 5);
LazinatorMemory multipleBufferResult = await BinaryTree.SerializeLazinatorAsync(options);
// Write to one or more blobs
var index = (useConsolidatedMemory || indices == null || !indices.Any()) ? new PersistentIndex(fullPath, blobManager, containedInSingleBlob) : new PersistentIndex(indices.Last());
await index.PersistLazinatorMemoryAsync(multipleBufferResult);
indices.Add(index);
if (recreateIndex)
{
index = await PersistentIndex.ReadFromBlobAsync(blobManager, fullPath, null, index.IndexVersion);
}
var revisedMemory = index.GetLazinatorMemory();
if (useConsolidatedMemory)
{
var consolidatedMemory = await revisedMemory.GetConsolidatedMemoryAsync();
revisedMemory = new LazinatorMemory(consolidatedMemory);
if (round != numRounds - 1)
{
indices.RemoveAt(0); // with consolidated memory, we're not using diff serialization
}
}
//Debug.WriteLine(revisedMemory.ToStringByChunk());
//Debug.WriteLine($"Consolidated{round}: " + revisedMemory.ToStringConsolidated());
BinaryTree = new LazinatorBinaryTree <WDouble>(revisedMemory);
round++;
});
DeleteChunksAndIndex(useConsolidatedMemory, indices, blobManager); // note that there is no DeleteAsync, so we use the same methods here
}
public void Add_WithNullIndexData_ThrowArgumentNullException()
{
//Arrange
var key = Guid.NewGuid().ToString();
IndexData indexData = null;
//Action
using (var index = new PersistentIndex(_indexFilePath, IndexBinTestCapacity))
{
index.Add(key, indexData);
}
//Assert
//Should throw ArgumentNullException
}
public void Add_WithNullKey_ThrowArgumentNullException()
{
//Arrange
var key = string.Empty;
var indexData = new IndexData();
//Action
using (var index = new PersistentIndex(_indexFilePath, IndexBinTestCapacity))
{
index.Add(key, indexData);
}
//Assert
//Should throw ArgumentNullException
}
public void Add_WithOverFreeDiskSpace_ThrowNotEnoughDiskSpaceException()
{
//Arrange
var bigCapacity = Sizes.Size1Tb * 10; //Assumes 10Tb will exceed any exists hard disk capacity
var indexData = new IndexData();
var key = Guid.NewGuid().ToString();
//Action
using (var storage = new PersistentIndex(_indexFilePath, bigCapacity))
{
storage.Add(key, indexData);
}
//Assert
//Should throw NotEnoughDiskSpaceException
}
public async Task BinaryTreeTest_DiffSerialization_RewritingAll_Async(bool useFile, bool containedInSingleBlob, bool recreateIndex, bool poolMemory)
{
// Here, we test making a change that in fact requires rewriting the entire Lazinator object. The change will be to the
// only node's Data, and the node has no Left or Right, so no references to the original data can be made.
var tree1 = new LazinatorBinaryTree <WByte>();
tree1.Add(1);
LazinatorMemory initialResult = await tree1.SerializeLazinatorAsync(LazinatorSerializationOptions.Default);
var tree2 = new LazinatorBinaryTree <WByte>(initialResult);
tree2.Root.Data = 2;
LazinatorSerializationOptions options = new LazinatorSerializationOptions(IncludeChildrenMode.IncludeAllChildren, false, false, true, 20);
LazinatorMemory afterChange = await tree2.SerializeLazinatorAsync(options);
IBlobManager blobManager = GetBlobManager(useFile, poolMemory);
string fullPath = GetPathForIndexAndBlobs(useFile, true);
if (fullPath == null)
{
return;
}
PersistentIndex index = new PersistentIndex(fullPath, blobManager, containedInSingleBlob);
await index.PersistLazinatorMemoryAsync(afterChange);
if (recreateIndex)
{
index = PersistentIndex.ReadFromBlob(blobManager, fullPath, null, index.IndexVersion);
}
var afterChangeReloaded = await index.GetLazinatorMemoryAsync();
var tree3 = new LazinatorBinaryTree <WByte>(afterChangeReloaded);
tree3.Root.Data.WrappedValue.Should().Be((byte)2);
var tree4 = new LazinatorBinaryTree <WByte>(afterChange);
tree4.Root.Data.WrappedValue.Should().Be((byte)2);
if (poolMemory)
{
blobManager.MemoryAllocator.FreeMemory(fullPath);
}
}
public void BinaryTreeTest_DiffSerialization_KeepingSomeInformation(bool useFile, bool containedInSingleBlob, bool recreateIndex)
{
// Here, we test making a change that in fact requires rewriting the entire Lazinator object. The change will be to the
// only node's Data, and the node has no Left or Right, so no references to the original data can be made.
var tree1 = new LazinatorBinaryTree <WByte>();
tree1.Add(1);
tree1.Add(3); // right node to above
LazinatorMemory initialResult = tree1.SerializeLazinator(LazinatorSerializationOptions.Default);
var tree2 = new LazinatorBinaryTree <WByte>(initialResult);
tree2.Root.Data = 2; // should now be 2 and 3
LazinatorSerializationOptions options = new LazinatorSerializationOptions(IncludeChildrenMode.IncludeAllChildren, false, false, true, 20);
LazinatorMemory afterChange = tree2.SerializeLazinator(options);
IBlobManager blobManager = useFile ? new FileBlobManager() : new InMemoryBlobManager();
string fullPath = GetPathForIndexAndBlobs(useFile, true);
PersistentIndex index = new PersistentIndex(fullPath, blobManager, containedInSingleBlob);
index.PersistLazinatorMemory(afterChange);
if (recreateIndex)
{
index = PersistentIndex.ReadFromBlob(blobManager, fullPath, null, index.IndexVersion);
}
var afterChangeReloaded = index.GetLazinatorMemory();
var tree3 = new LazinatorBinaryTree <WByte>(afterChangeReloaded);
tree3.Root.Data.WrappedValue.Should().Be((byte)2);
tree3.Root.RightNode.Data.WrappedValue.Should().Be((byte)3);
var tree4 = new LazinatorBinaryTree <WByte>(afterChange);
tree4.Root.Data.WrappedValue.Should().Be((byte)2);
tree4.Root.RightNode.Data.WrappedValue.Should().Be((byte)3);
}
public void PersistentIndexForking()
{
InMemoryBlobManager blobManager = new InMemoryBlobManager();
LazinatorSerializationOptions options = new LazinatorSerializationOptions(IncludeChildrenMode.IncludeAllChildren, false, false, true, 20, 5);
// let's make a plan of what data should be added to a list of doubles at each point of our persistent index fork
ForkPlan root = new ForkPlan(null, null, new List <double> {
1.0, 2.0
});
ForkPlan child1 = new ForkPlan(root, new List <int> {
1
}, new List <double>()
{
3.0
});
ForkPlan child2 = new ForkPlan(root, new List <int> {
2
}, new List <double>()
{
4.0, 5.0
});
ForkPlan child3 = new ForkPlan(root, new List <int> {
3
}, new List <double>()
{
6.0, 7.0
});
ForkPlan child2_1 = new ForkPlan(child2, new List <int> {
2, 1
}, new List <double>()
{
8.0, 9.0
});
ForkPlan child2_2 = new ForkPlan(child2, new List <int> {
2, 2
}, new List <double>()
{
8.0, 9.0, 10.0
});
ForkPlan child2_2_1 = new ForkPlan(child2_2, new List <int> {
2, 2, 1
}, new List <double>()
{
11.0
});
void PersistAccordingToPlan(PersistentIndex index, ForkPlan plan)
{
if (index.PreviousPersistentIndex == null)
{
BinaryTree = new LazinatorBinaryTree <WDouble>();
}
else
{
var loadedFromPrevious = index.PreviousPersistentIndex.GetLazinatorMemory();
BinaryTree = new LazinatorBinaryTree <WDouble>(loadedFromPrevious);
}
foreach (var item in plan.DataToAdd)
{
BinaryTree.Add(item);
}
LazinatorMemory serialized = BinaryTree.SerializeLazinator(options);
index.PersistLazinatorMemory(serialized);
}
PersistentIndex index_root = new PersistentIndex("forktest", blobManager, false);
PersistAccordingToPlan(index_root, root);
PersistentIndex index_child1 = new PersistentIndex(index_root, 1);
PersistAccordingToPlan(index_child1, child1);
PersistentIndex index_child2 = new PersistentIndex(index_root, 2);
PersistAccordingToPlan(index_child2, child2);
PersistentIndex index_child3 = new PersistentIndex(index_root, 3);
PersistAccordingToPlan(index_child3, child3);
PersistentIndex index_child2_1 = new PersistentIndex(index_child2, 1);
PersistAccordingToPlan(index_child2_1, child2_1);
PersistentIndex index_child2_2 = new PersistentIndex(index_child2, 2);
PersistAccordingToPlan(index_child2_2, child2_2);
PersistentIndex index_child2_2_1 = new PersistentIndex(index_child2_2, 1);
PersistAccordingToPlan(index_child2_2_1, child2_2_1);
void VerifyProperPersistence(PersistentIndex index, ForkPlan plan)
{
var loaded = index.GetLazinatorMemory();
BinaryTree = new LazinatorBinaryTree <WDouble>(loaded);
var treeContents = BinaryTree.Select(x => (double)x).ToList();
var expectedContents = plan.OverallData().ToList();
treeContents.SequenceEqual(expectedContents).Should().BeTrue();
}
// verify contents
VerifyProperPersistence(index_root, root);
VerifyProperPersistence(index_child1, child1);
VerifyProperPersistence(index_child2, child2);
VerifyProperPersistence(index_child3, child3);
VerifyProperPersistence(index_child2_1, child2_1);
VerifyProperPersistence(index_child2_2, child2_2);
VerifyProperPersistence(index_child2_2_1, child2_2_1);
// now verify contents after progressive deletion. When an index has forks from it, we will only delete things that we know are safe to delete.
// This illustrates the proper approach. When we are done with a version from which there are forks, we can delete newly omitted memory chunks (not relevant for root), plus the index itself. When we are done with a version that has no forks, but is itself forked from something else, we can delete previously included, newly omitted, and newly included, but not before the last fork. If it's the last one, though, we make it so that all previously included chunks are deleted, including those preceding the last fork.
index_root.DeleteIndex();
VerifyProperPersistence(index_child1, child1);
index_child1.Delete(PersistentIndexMemoryChunkStatus.PreviouslyIncluded, false);
index_child1.Delete(PersistentIndexMemoryChunkStatus.NewlyOmitted, false);
index_child1.Delete(PersistentIndexMemoryChunkStatus.NewlyIncluded, false);
index_child1.DeleteIndex();
VerifyProperPersistence(index_child2, child2);
index_child2.Delete(PersistentIndexMemoryChunkStatus.NewlyOmitted, true); // only newly omitted, because there are forks
index_child2.DeleteIndex();
VerifyProperPersistence(index_child3, child3);
index_child3.Delete(PersistentIndexMemoryChunkStatus.PreviouslyIncluded, false);
index_child3.Delete(PersistentIndexMemoryChunkStatus.NewlyOmitted, false);
index_child3.Delete(PersistentIndexMemoryChunkStatus.NewlyIncluded, false);
index_child3.DeleteIndex();
VerifyProperPersistence(index_child2_1, child2_1);
index_child2_1.Delete(PersistentIndexMemoryChunkStatus.PreviouslyIncluded, false);
index_child2_1.Delete(PersistentIndexMemoryChunkStatus.NewlyOmitted, false);
index_child2_1.Delete(PersistentIndexMemoryChunkStatus.NewlyIncluded, false);
index_child2_1.DeleteIndex();
VerifyProperPersistence(index_child2_2, child2_2);
index_child2_2.Delete(PersistentIndexMemoryChunkStatus.NewlyOmitted, true); // only newly omitted, because there are forks
index_child2_2.DeleteIndex();
VerifyProperPersistence(index_child2_2_1, child2_2_1);
index_child2_2_1.Delete(PersistentIndexMemoryChunkStatus.PreviouslyIncluded, true);
index_child2_2_1.Delete(PersistentIndexMemoryChunkStatus.NewlyOmitted, false);
index_child2_2_1.Delete(PersistentIndexMemoryChunkStatus.NewlyIncluded, false);
index_child2_2_1.DeleteIndex();
blobManager.Storage.Any().Should().BeFalse();
}
