private static int ReadLastIdFromTable(Table table, SnapshotReader snapshot)
{
using (var iterator = table.Iterate(snapshot, null))
{
if (!iterator.Seek(Slice.AfterAllKeys))
return 0;
var id = iterator.CurrentKey.ToString();
return int.Parse(id);
}
}
public bool Contains(SnapshotReader snapshot, Slice key, WriteBatch writeBatch, out ushort? version)
{
return snapshot.Contains(TableName, key, out version, writeBatch);
}
public bool Contains(SnapshotReader snapshot, Slice key, WriteBatch writeBatch)
{
ushort? version;
return Contains(snapshot, key, writeBatch, out version);
}
public virtual IIterator Iterate(SnapshotReader snapshot, WriteBatch writeBatch)
{
return snapshot.Iterate(TableName);
}
public virtual IIterator MultiRead(SnapshotReader snapshot, Slice key)
{
return snapshot.MultiRead(TableName, key);
}
public virtual ReadResult Read(SnapshotReader snapshot, Slice key, WriteBatch writeBatch)
{
return snapshot.Read(TableName, key, writeBatch);
}
//for debugging purposes
public Dictionary<string, string> Dump(SnapshotReader snapshot)
{
using (var iterator = snapshot.Iterate(TableName))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
return new Dictionary<string, string>();
var results = new Dictionary<string, string>();
do
{
bool isMultiTreeKey;
using (var multiIterator = snapshot.MultiRead(TableName, iterator.CurrentKey))
{
if (!multiIterator.Seek(Slice.BeforeAllKeys))
{
isMultiTreeKey = false;
}
else
{
isMultiTreeKey = true;
const string subtreeKeyPrefix = "[sub tree val: ]";
do
{
results.Add(subtreeKeyPrefix + iterator.CurrentKey + " " + results.Count , new StreamReader(multiIterator.CreateReaderForCurrent().AsStream()).ReadToEnd());
} while (multiIterator.MoveNext());
}
}
if(!isMultiTreeKey)
results.Add(iterator.CurrentKey.ToString(), new StreamReader(iterator.CreateReaderForCurrent().AsStream()).ReadToEnd());
} while (iterator.MoveNext());
return results;
}
}
public virtual ushort? ReadVersion(SnapshotReader snapshot, Slice key, WriteBatch writeBatch)
{
return snapshot.ReadVersion(TableName, key, writeBatch);
}
public int GetDataSize(SnapshotReader snapshot, Slice key)
{
return snapshot.GetDataSize(TableName, key);
}
public StorageActionsAccessor(TableStorage storage, Reference<WriteBatch> writeBatch, SnapshotReader snapshot, IdGenerator generator, IBufferPool bufferPool)
: base(snapshot, generator, bufferPool)
{
this.storage = storage;
this.writeBatch = writeBatch;
}
public void ShouldReusePagesButStillNotMixStuffUp()
{
var random = new Random(1234);
var buffer = new byte[1024 * 512];
random.NextBytes(buffer);
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
Env.CreateTree(tx, "ATree");
Env.CreateTree(tx, "BTree");
tx.Commit();
}
var generator = new Random(1000);
var bTreeValues = new SortedDictionary<int, int>();
// Starting the tree.
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
var aTree = tx.Environment.CreateTree(tx, "ATree");
var bTree = tx.Environment.CreateTree(tx, "BTree");
for (int i = 0; i < 10000; i++)
{
aTree.Add("A" + generator.Next(), new byte[0]);
int bKey = generator.Next();
bTree.Add("B" + bKey, new byte[0]);
bTreeValues[bKey] = bKey;
}
tx.Commit();
}
for ( int iterations = 0; iterations < 1000; iterations++ )
{
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
var aTree = tx.Environment.CreateTree(tx, "ATree");
var bTree = tx.Environment.CreateTree(tx, "BTree");
if (generator.Next(3) == 0)
{
// The rare operation is to delete.
// We dont have enough to actually make sense to delete.
if (bTreeValues.Count < 200)
continue;
// We always delete from B
int i = generator.Next(bTreeValues.Count - 100);
var valuesToIterate = new List<int>(bTreeValues.Keys.Skip(i).Take(100));
foreach ( int value in valuesToIterate)
{
bTree.Delete("B" + value);
bTreeValues.Remove(value);
}
}
else if (generator.Next(2) == 0)
{
// Add on A
for (int i = 0; i < 100; i++)
{
aTree.Add("A" + generator.Next(), new byte[0]);
}
}
else
{
// Add on B
for (int i = 0; i < 100; i++)
{
int bKey = generator.Next();
if (!bTreeValues.ContainsKey(bKey))
{
bTree.Add("B" + bKey, new byte[0]);
bTreeValues[bKey] = bKey;
}
}
}
tx.Commit();
}
}
using (var tx = Env.NewTransaction(TransactionFlags.Read))
{
var aTree = tx.Environment.CreateTree(tx, "ATree");
var startWithValue = new Slice("A");
var reader = new SnapshotReader(tx);
using (var iterator = reader.Iterate("ATree"))
{
iterator.Seek(Slice.BeforeAllKeys);
do
{
if (iterator.CurrentKey != Slice.AfterAllKeys && iterator.CurrentKey != Slice.BeforeAllKeys)
{
Assert.True(iterator.CurrentKey.StartsWith(startWithValue));
}
}
while (iterator.MoveNext());
}
}
}
private void ValidateLotsOfTestDataForTree(SnapshotReader snapshot, string treeName)
{
for (int i = 0; i < 50; i++)
{
for (int j = 0; j < 500; j++)
{
var index = (i + "/ " + j);
var key = "key/" + index;
var expectedValue = "value/" + index;
var result = snapshot.Read(treeName, key);
Assert.NotNull(result);
var fetchedValue = Encoding.UTF8.GetString(result.Reader.AsStream().ReadData());
Assert.Equal(expectedValue, fetchedValue);
}
}
}
private void DeleteTermsForIndexEntry(SnapshotReader snapshot, ValueReader reader, long id, int fieldId)
{
var termBuffer = _bufferPool.Take(reader.Length);
_usedBuffers.Add(termBuffer);
reader.Read(termBuffer, 0, reader.Length);
var termSlice = new Slice(termBuffer, (ushort) reader.Length);
var tree = GetTreeName(fieldId);
using (var termIt = snapshot.MultiRead(tree, termSlice))
{
var currentFieldDocument = _bufferPool.Take(FullTextIndex.FieldDocumentSize);
try
{
if (termIt.Seek(new Slice(currentFieldDocument)) == false)
return;
do
{
termIt.CurrentKey.CopyTo(currentFieldDocument);
if (EndianBitConverter.Big.ToInt64(currentFieldDocument, 0) != id)
break;
var valueBuffer = _bufferPool.Take(termIt.CurrentKey.Size);
_usedBuffers.Add(valueBuffer);
termIt.CurrentKey.CopyTo(valueBuffer);
_writeBatch.MultiDelete(termSlice, new Slice(valueBuffer), tree);
} while (termIt.MoveNext());
}
finally
{
_bufferPool.Return(currentFieldDocument);
}
}
}
protected StorageActionsBase(SnapshotReader snapshot, IdGenerator idGenerator, IBufferPool bufferPool)
{
this.bufferPool = bufferPool;
Snapshot = snapshot;
IdGenerator = idGenerator;
}
public void FaultyOverflowPagesHandling_CannotModifyReadOnlyPages(int initialNumberOfDocs, int numberOfModifications, int seed)
{
const string documents = "documents";
const string keyByEtag = "documents_key_by_etag";
const string metadata = "documents_metadata";
var inMemoryKeysByEtag = new Dictionary<Guid, string>();
var inMemoryKeys = new HashSet<string>();
var r = new Random(seed);
var uuidGenerator = new UuidGenerator();
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
var docsTree = Env.CreateTree(tx, documents);
var metadataTree = Env.CreateTree(tx, metadata);
var indexTree = Env.CreateTree(tx, keyByEtag);
for (int i = 0; i < initialNumberOfDocs; i++)
{
var etag = uuidGenerator.CreateSequentialUuid();
var docKey = get_id(etag, r);
put_doc(r, etag, inMemoryKeysByEtag, inMemoryKeys, docKey, docsTree, metadataTree, indexTree);
}
tx.Commit();
}
for (int i = 0; i < numberOfModifications; i++)
{
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
var docsTree = tx.ReadTree(documents);
var metadataTree = tx.ReadTree(metadata);
var indexTree = tx.ReadTree(keyByEtag);
if (r.Next(3) == 0)
{
// insert new
var etag = uuidGenerator.CreateSequentialUuid();
var docKey = get_id(etag, r);
put_doc(r, etag, inMemoryKeysByEtag, inMemoryKeys, docKey, docsTree, metadataTree, indexTree);
}
else
{
// update existing
var docCount = inMemoryKeysByEtag.Values.Count;
var docKeyToUpdate = inMemoryKeysByEtag.Values.Skip(r.Next(0, docCount - 1)).First();
var etag = uuidGenerator.CreateSequentialUuid();
put_doc(r, etag, inMemoryKeysByEtag, inMemoryKeys, docKeyToUpdate, docsTree, metadataTree, indexTree);
}
tx.Commit();
}
}
using (var tx = Env.NewTransaction(TransactionFlags.Read))
{
var docsTree = tx.ReadTree(documents);
var metadataTree = tx.ReadTree(metadata);
var reader = new SnapshotReader(tx);
var count = 0;
using (var iterator = reader.Iterate(keyByEtag))
{
iterator.Seek(Slice.BeforeAllKeys);
do
{
var etag = Guid.Parse(iterator.CurrentKey.ToString());
string key;
using (var currentDataStream = iterator.CreateReaderForCurrent().AsStream())
{
var keyBytes = currentDataStream.ReadData();
key = Encoding.UTF8.GetString(keyBytes);
}
var inMemoryKey = inMemoryKeysByEtag[etag];
Assert.Equal(inMemoryKey, key);
var docReadResult = docsTree.Read(key);
Assert.NotNull(docReadResult);
var metadataReader = metadataTree.Read(key).Reader;
Assert.NotNull(metadataReader);
var etagFromMetadata = new byte[16];
metadataReader.Read(etagFromMetadata, 0, 16);
var readEtag = new Guid(etagFromMetadata);
if (etag != readEtag)
{
string existingDocKey;
if (inMemoryKeysByEtag.TryGetValue(readEtag, out existingDocKey))
{
Console.WriteLine("Etag " + readEtag + " belongs to " + existingDocKey + " document");
}
else
{
Console.WriteLine("There is no document with etag " + readEtag);
}
}
Assert.Equal(etag, readEtag);
count++;
}
while (iterator.MoveNext());
}
Assert.Equal(inMemoryKeysByEtag.Count, count);
}
}
