private void ReplayWriteAction(WriteActivityEntry activityEntry, ref Transaction tx)
{
var tree = tx.ReadTree(activityEntry.TreeName);
switch (activityEntry.ActionType)
{
case DebugActionType.Add:
tree.Add(activityEntry.Key, activityEntry.ValueStream);
break;
case DebugActionType.Delete:
tree.Delete(activityEntry.Key);
break;
case DebugActionType.MultiAdd:
tree.MultiAdd(activityEntry.Key, new Slice(Encoding.UTF8.GetBytes(activityEntry.Value.ToString())));
break;
case DebugActionType.MultiDelete:
tree.MultiDelete(activityEntry.Key, new Slice(Encoding.UTF8.GetBytes(activityEntry.Value.ToString())));
break;
case DebugActionType.CreateTree:
_env.CreateTree(tx, activityEntry.TreeName);
break;
case DebugActionType.Increment:
var buffer = new byte[sizeof(long)];
activityEntry.ValueStream.Read(buffer, 0, buffer.Length);
var delta = EndianBitConverter.Little.ToInt64(buffer, 0);
tree.Increment(activityEntry.Key, delta);
break;
case DebugActionType.AddStruct:
tree.Add(activityEntry.Key, activityEntry.ValueStream);
break;
case DebugActionType.RenameTree:
_env.RenameTree(tx, activityEntry.TreeName, activityEntry.Key.ToString());
break;
default: //precaution against newly added action types
throw new InvalidOperationException("unsupported tree action type: " + activityEntry.ActionType);
}
}
public void Record_debug_journal_and_replay_it()
{
var structSchema = new StructureSchema <SampleStruct>()
.Add <int>(SampleStruct.Foo)
.Add <string>(SampleStruct.Bar);
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = new DebugJournal(debugJouralName, env, true);
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("{ \"title\": \"foo\",\"name\":\"bar\"}"));
writeBatch.Add("foo", valueBuffer, "test-tree");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2!"));
writeBatch.Add("bar", valueBuffer, "test-tree");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2 3!"));
writeBatch.Add("foo-bar", valueBuffer, "test-tree");
writeBatch.MultiAdd("multi-foo", "AA", "test-tree");
env.Writer.Write(writeBatch);
}
using (var writeBatch = new WriteBatch())
{
writeBatch.Increment("incr-key", 5, "test-tree");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
Assert.Equal(5, tx.ReadTree("test-tree").Read("incr-key").Reader.ReadLittleEndianInt64());
using (var writeBatch = new WriteBatch())
{
writeBatch.Increment("incr-key", 5, "test-tree");
env.Writer.Write(writeBatch);
}
Assert.Equal(5, tx.ReadTree("test-tree").Read("incr-key").Reader.ReadLittleEndianInt64());
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
Assert.Equal(10, tx.ReadTree("test-tree").Read("incr-key").Reader.ReadLittleEndianInt64());
}
using (var writeBatch = new WriteBatch())
{
writeBatch.MultiAdd("multi-foo", "BB", "test-tree");
writeBatch.MultiAdd("multi-foo", "CC", "test-tree");
writeBatch.Delete("foo-bar", "test-tree");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree2");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1!"));
writeBatch.Add("foo", valueBuffer, "test-tree2");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2!"));
writeBatch.Add("bar", valueBuffer, "test-tree2");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2 3!"));
writeBatch.Add("foo-bar", valueBuffer, "test-tree2");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "structures-tree");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
writeBatch.AddStruct("structs/1", new Structure <SampleStruct>(structSchema)
.Set(SampleStruct.Foo, 13)
.Set(SampleStruct.Bar, "debug journal testing"),
"structures-tree");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "rename-me");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
writeBatch.Add("item", "renaming tree test", "rename-me");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.RenameTree(tx, "rename-me", "renamed");
tx.Commit();
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = DebugJournal.FromFile(debugJouralName, env);
env.DebugJournal.Replay();
using (var snapshot = env.CreateSnapshot())
{
Assert.Equal("{ \"title\": \"foo\",\"name\":\"bar\"}", snapshot.Read("test-tree", "foo").Reader.ToStringValue());
Assert.Equal("testing testing 1 2!", snapshot.Read("test-tree", "bar").Reader.ToStringValue());
Assert.Equal("testing testing 1!", snapshot.Read("test-tree2", "foo").Reader.ToStringValue());
Assert.Equal("testing testing 1 2!", snapshot.Read("test-tree2", "bar").Reader.ToStringValue());
Assert.Equal("testing testing 1 2 3!", snapshot.Read("test-tree2", "foo-bar").Reader.ToStringValue());
Assert.Equal(10, snapshot.Read("test-tree", "incr-key").Reader.ReadLittleEndianInt64());
Assert.Equal(0, snapshot.ReadVersion("test-tree", "foo-bar"));
using (var iter = snapshot.MultiRead("test-tree", "multi-foo"))
{
iter.Seek(Slice.BeforeAllKeys);
Assert.Equal("AA", iter.CurrentKey.ToString());
Assert.DoesNotThrow(() => iter.MoveNext());
Assert.Equal("BB", iter.CurrentKey.ToString());
Assert.DoesNotThrow(() => iter.MoveNext());
Assert.Equal("CC", iter.CurrentKey.ToString());
}
var structReader = snapshot.ReadStruct("structures-tree", "structs/1", structSchema).Reader;
Assert.Equal(13, structReader.ReadInt(SampleStruct.Foo));
Assert.Equal("debug journal testing", structReader.ReadString(SampleStruct.Bar));
Assert.Equal("renaming tree test", snapshot.Read("renamed", "item").Reader.ToStringValue());
}
}
}
private static void Migrate(StorageEnvironment env, string tableName, Action <string> output, Action <Slice, RavenJObject> modifyRecord)
{
long entriesCount;
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
entriesCount = tx.ReadTree(tableName).State.EntriesCount;
}
if (entriesCount == 0)
{
output(string.Format("No records to migrate in '{0}' table.", tableName));
return;
}
output(string.Format("Starting to migrate '{0}' table to. Records to process: {1}", tableName, entriesCount));
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.DeleteTree(txw, "Temp_" + tableName);
txw.Commit();
}
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(txw, "Temp_" + tableName);
txw.Commit();
}
var migrated = 0L;
var keyToSeek = Slice.BeforeAllKeys;
do
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var destTree = txw.ReadTree("Temp_" + tableName);
var srcTree = txw.ReadTree(tableName);
var iterator = srcTree.Iterate();
if (iterator.Seek(keyToSeek) == false)
{
break;
}
var itemsInBatch = 0;
do
{
keyToSeek = iterator.CurrentKey;
if (itemsInBatch != 0 && itemsInBatch % 100 == 0)
{
break;
}
using (var stream = iterator.CreateReaderForCurrent().AsStream())
{
var value = stream.ToJObject();
modifyRecord(iterator.CurrentKey, value);
using (var streamValue = new MemoryStream())
{
value.WriteTo(streamValue);
streamValue.Position = 0;
destTree.Add(iterator.CurrentKey, streamValue);
}
migrated++;
itemsInBatch++;
}
} while (iterator.MoveNext());
txw.Commit();
output(string.Format("{0} of {1} entries processed.", migrated, entriesCount));
}
} while (migrated < entriesCount);
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.DeleteTree(txw, tableName);
env.RenameTree(txw, "Temp_" + tableName, tableName);
txw.Commit();
}
}
public void Record_debug_journal_and_replay_it()
{
var structSchema = new StructureSchema<SampleStruct>()
.Add<int>(SampleStruct.Foo)
.Add<string>(SampleStruct.Bar);
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = new DebugJournal(debugJouralName, env, true);
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("{ \"title\": \"foo\",\"name\":\"bar\"}"));
writeBatch.Add("foo", valueBuffer, "test-tree");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2!"));
writeBatch.Add("bar", valueBuffer, "test-tree");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2 3!"));
writeBatch.Add("foo-bar", valueBuffer, "test-tree");
writeBatch.MultiAdd("multi-foo", "AA", "test-tree");
env.Writer.Write(writeBatch);
}
using (var writeBatch = new WriteBatch())
{
writeBatch.Increment("incr-key", 5, "test-tree");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
Assert.Equal(5, tx.ReadTree("test-tree").Read("incr-key").Reader.ReadLittleEndianInt64());
using (var writeBatch = new WriteBatch())
{
writeBatch.Increment("incr-key", 5, "test-tree");
env.Writer.Write(writeBatch);
}
Assert.Equal(5, tx.ReadTree("test-tree").Read("incr-key").Reader.ReadLittleEndianInt64());
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
Assert.Equal(10, tx.ReadTree("test-tree").Read("incr-key").Reader.ReadLittleEndianInt64());
}
using (var writeBatch = new WriteBatch())
{
writeBatch.MultiAdd("multi-foo", "BB", "test-tree");
writeBatch.MultiAdd("multi-foo", "CC", "test-tree");
writeBatch.Delete("foo-bar", "test-tree");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree2");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1!"));
writeBatch.Add("foo", valueBuffer, "test-tree2");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2!"));
writeBatch.Add("bar", valueBuffer, "test-tree2");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2 3!"));
writeBatch.Add("foo-bar", valueBuffer, "test-tree2");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "structures-tree");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
writeBatch.AddStruct("structs/1", new Structure<SampleStruct>(structSchema)
.Set(SampleStruct.Foo, 13)
.Set(SampleStruct.Bar, "debug journal testing"),
"structures-tree");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "rename-me");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
writeBatch.Add("item", "renaming tree test", "rename-me");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.RenameTree(tx, "rename-me", "renamed");
tx.Commit();
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = DebugJournal.FromFile(debugJouralName, env);
env.DebugJournal.Replay();
using (var snapshot = env.CreateSnapshot())
{
Assert.Equal("{ \"title\": \"foo\",\"name\":\"bar\"}", snapshot.Read("test-tree", "foo").Reader.ToStringValue());
Assert.Equal("testing testing 1 2!", snapshot.Read("test-tree", "bar").Reader.ToStringValue());
Assert.Equal("testing testing 1!", snapshot.Read("test-tree2", "foo").Reader.ToStringValue());
Assert.Equal("testing testing 1 2!", snapshot.Read("test-tree2", "bar").Reader.ToStringValue());
Assert.Equal("testing testing 1 2 3!", snapshot.Read("test-tree2", "foo-bar").Reader.ToStringValue());
Assert.Equal(10, snapshot.Read("test-tree", "incr-key").Reader.ReadLittleEndianInt64());
Assert.Equal(0,snapshot.ReadVersion("test-tree","foo-bar"));
using (var iter = snapshot.MultiRead("test-tree","multi-foo"))
{
iter.Seek(Slice.BeforeAllKeys);
Assert.Equal("AA",iter.CurrentKey.ToString());
Assert.DoesNotThrow(() => iter.MoveNext());
Assert.Equal("BB",iter.CurrentKey.ToString());
Assert.DoesNotThrow(() => iter.MoveNext());
Assert.Equal("CC",iter.CurrentKey.ToString());
}
var structReader = snapshot.ReadStruct("structures-tree", "structs/1", structSchema).Reader;
Assert.Equal(13, structReader.ReadInt(SampleStruct.Foo));
Assert.Equal("debug journal testing", structReader.ReadString(SampleStruct.Bar));
Assert.Equal("renaming tree test", snapshot.Read("renamed", "item").Reader.ToStringValue());
}
}
}
public void MigrateToStructures <T>(StorageEnvironment env, TableOfStructures <T> table, Action <string> output, Action <RavenJObject, Structure <T> > copyToStructure)
{
long entriesCount;
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
entriesCount = tx.ReadTree(table.TableName).State.EntriesCount;
}
if (entriesCount == 0)
{
output(string.Format("No records to migrate in '{0}' table.", table.TableName));
return;
}
output(string.Format("Starting to migrate '{0}' table to use structures. Records to process: {1}", table.TableName, entriesCount));
var migratedEntries = 0L;
var keyToSeek = Slice.BeforeAllKeys;
// delete Temp_TabelName if exists
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.DeleteTree(txw, "Temp_" + table.TableName);
txw.Commit();
}
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(txw, "Temp_" + table.TableName);
txw.Commit();
}
do
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var destTree = txw.ReadTree("Temp_" + table.TableName);
var srcTree = txw.ReadTree(table.TableName);
var iterator = srcTree.Iterate();
if (iterator.Seek(keyToSeek) == false)
{
break;
}
var writtenStructsSize = 0;
do
{
keyToSeek = iterator.CurrentKey;
if (writtenStructsSize > 8 * 1024 * 1024) // 8 MB
{
break;
}
var readerForCurrent = iterator.CreateReaderForCurrent();
using (var stream = readerForCurrent.AsStream())
{
var currentDataSize = iterator.GetCurrentDataSize();
try
{
RavenJObject jsonValue = stream.ToJObject();
var structValue = new Structure <T>(table.Schema);
copyToStructure(jsonValue, structValue);
destTree.WriteStruct(iterator.CurrentKey, structValue);
writtenStructsSize += structValue.GetSize();
}
catch (Exception)
{
// already converted this, probably, just move as is
int used;
var readBytes = readerForCurrent.ReadBytes(currentDataSize, out used);
destTree.Add(iterator.CurrentKey, readBytes);
writtenStructsSize += currentDataSize;
}
migratedEntries++;
}
} while (iterator.MoveNext());
txw.Commit();
output(string.Format("{0} of {1} records processed.", migratedEntries, entriesCount));
}
} while (migratedEntries < entriesCount);
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.DeleteTree(txw, table.TableName);
env.RenameTree(txw, "Temp_" + table.TableName, table.TableName);
txw.Commit();
}
output(string.Format("All records of '{0}' table have been migrated to structures.", table.TableName));
}