private void Restore(StorageEnvironment env, IEnumerable <ZipArchiveEntry> entries)
{
using (env.Journal.Applicator.TakeFlushingLock())
{
env.FlushLogToDataFile();
var transactionPersistentContext = new TransactionPersistentContext(true);
using (var txw = env.NewLowLevelTransaction(transactionPersistentContext, TransactionFlags.ReadWrite))
{
var toDispose = new List <IDisposable>();
var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;
var tempDirSettings = new VoronPathSetting(tempDir);
Restore(env, entries, tempDirSettings, toDispose, txw);
}
}
}
public unsafe void AssertSubscriptionIdExists(long id)
{
var transactionPersistentContext = new TransactionPersistentContext();
using (var tx = _environment.ReadTransaction(transactionPersistentContext))
{
var table = tx.OpenTable(_subscriptionsSchema, SubscriptionSchema.SubsTree);
var subscriptionId = Bits.SwapBytes((ulong)id);
Slice subsriptionSlice;
using (Slice.External(tx.Allocator, (byte *)&subscriptionId, sizeof(long), out subsriptionSlice))
{
if (table.VerifyKeyExists(subsriptionSlice) == false)
{
throw new SubscriptionDoesNotExistException(
"There is no subscription configuration for specified identifier (id: " + id + ")");
}
}
}
}
public void GetRunningSusbscriptions(BlittableJsonTextWriter writer,
DocumentsOperationContext context, int start, int take)
{
var transactionPersistentContext = new TransactionPersistentContext();
using (var tx = _environment.ReadTransaction(transactionPersistentContext))
{
var connections = new List <DynamicJsonValue>(take);
var skipped = 0;
var taken = 0;
foreach (var kvp in _subscriptionStates)
{
var subscriptionState = kvp.Value;
var subscriptionId = kvp.Key;
if (taken > take)
{
break;
}
if (subscriptionState?.Connection == null)
{
continue;
}
if (skipped < start)
{
skipped++;
continue;
}
var subscriptionData = ExtractSubscriptionConfigValue(GetSubscriptionConfig(subscriptionId, tx), context);
SetSubscriptionStateData(subscriptionState, subscriptionData);
connections.Add(subscriptionData);
taken++;
}
context.Write(writer, new DynamicJsonArray(connections));
writer.Flush();
}
}
public unsafe long CreateSubscription(BlittableJsonReaderObject criteria, long ackEtag = 0)
{
// Validate that this can be properly parsed into a criteria object
// and doing that without holding the tx lock
JsonDeserializationServer.SubscriptionCriteria(criteria);
var transactionPersistentContext = new TransactionPersistentContext();
using (var tx = _environment.WriteTransaction(transactionPersistentContext))
{
var table = tx.OpenTable(_subscriptionsSchema, SubscriptionSchema.SubsTree);
var subscriptionsTree = tx.ReadTree(SubscriptionSchema.IdsTree);
var id = subscriptionsTree.Increment(SubscriptionSchema.Id, 1);
long timeOfSendingLastBatch = 0;
long timeOfLastClientActivity = 0;
var bigEndianId = Bits.SwapBytes((ulong)id);
table.Insert(new TableValueBuilder
{
bigEndianId,
{ criteria.BasePointer, criteria.Size },
ackEtag,
timeOfSendingLastBatch,
timeOfLastClientActivity
});
tx.Commit();
if (_logger.IsInfoEnabled)
{
_logger.Info($"New Subscription With ID {id} was created");
}
return(id);
}
}
private static void Backup(
StorageEnvironment env, CompressionLevel compression, Action <string> infoNotify,
Action backupStarted, AbstractPager dataPager, ZipArchive package, string basePath, DataCopier copier)
{
var usedJournals = new List <JournalFile>();
long lastWrittenLogPage = -1;
long lastWrittenLogFile = -1;
LowLevelTransaction txr = null;
try
{
long allocatedPages;
var writePesistentContext = new TransactionPersistentContext(true);
var readPesistentContext = new TransactionPersistentContext(true);
using (var txw = env.NewLowLevelTransaction(writePesistentContext, TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
{
txr = env.NewLowLevelTransaction(readPesistentContext, TransactionFlags.Read); // now have snapshot view
allocatedPages = dataPager.NumberOfAllocatedPages;
Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);
infoNotify("Voron copy headers for " + basePath);
VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options, basePath);
// journal files snapshot
var files = env.Journal.Files; // thread safety copy
JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1;
journalNum <= journalInfo.CurrentJournal;
journalNum++)
{
var journalFile = files.FirstOrDefault(x => x.Number == journalNum);
// first check journal files currently being in use
if (journalFile == null)
{
long journalSize;
using (var pager = env.Options.OpenJournalPager(journalNum))
{
journalSize = Bits.NextPowerOf2(pager.NumberOfAllocatedPages * env.Options.PageSize);
}
journalFile = new JournalFile(env, env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
}
journalFile.AddRef();
usedJournals.Add(journalFile);
}
if (env.Journal.CurrentFile != null)
{
lastWrittenLogFile = env.Journal.CurrentFile.Number;
lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1;
}
// txw.Commit(); intentionally not committing
}
backupStarted?.Invoke();
// data file backup
var dataPart = package.CreateEntry(Path.Combine(basePath, Constants.DatabaseFilename), compression);
Debug.Assert(dataPart != null);
if (allocatedPages > 0) //only true if dataPager is still empty at backup start
{
using (var dataStream = dataPart.Open())
{
// now can copy everything else
copier.ToStream(dataPager, 0, allocatedPages, dataStream);
}
}
try
{
foreach (JournalFile journalFile in usedJournals)
{
var entryName = Path.Combine(basePath, StorageEnvironmentOptions.JournalName(journalFile.Number));
var journalPart = package.CreateEntry(entryName, compression);
Debug.Assert(journalPart != null);
long pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
if (journalFile.Number == lastWrittenLogFile)
{
pagesToCopy = lastWrittenLogPage + 1;
}
using (var stream = journalPart.Open())
{
copier.ToStream(env, journalFile, 0, pagesToCopy, stream);
infoNotify(string.Format("Voron copy journal file {0}", entryName));
}
}
}
finally
{
foreach (var journalFile in usedJournals)
{
journalFile.Release();
}
}
}
finally
{
txr?.Dispose();
}
}
private static long CopyFixedSizeTreeFromRoot(StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, Transaction txr,
TreeIterator rootIterator, string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context, CancellationToken token)
{
var treeNameSlice = rootIterator.CurrentKey.Clone(txr.Allocator);
var header = (FixedSizeTreeHeader.Embedded *)txr.LowLevelTransaction.RootObjects.DirectRead(treeNameSlice);
var fst = txr.FixedTreeFor(treeNameSlice, header->ValueSize);
Report(copiedTrees, totalTreesCount, 0, fst.NumberOfEntries, progressReport, $"Copying fixed size tree '{treeName}'. Progress: 0/{fst.NumberOfEntries} entries.", treeName);
CopyFixedSizeTree(fst, txw => txw.FixedTreeFor(treeNameSlice, header->ValueSize), compactedEnv, context, copiedEntries =>
{
Report(copiedTrees, totalTreesCount, copiedEntries, fst.NumberOfEntries, progressReport,
$"Copying fixed size tree '{treeName}'. Progress: {copiedEntries}/{fst.NumberOfEntries} entries.", treeName);
}, () =>
{
copiedTrees++;
Report(copiedTrees, totalTreesCount, fst.NumberOfEntries, fst.NumberOfEntries, progressReport,
$"Finished copying fixed size tree '{treeName}'. {fst.NumberOfEntries} entries copied.", treeName);
}, token);
return(copiedTrees);
}
private static long CopyVariableSizeTree(StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, Transaction txr, string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context, CancellationToken token)
{
var existingTree = txr.ReadTree(treeName);
Report(copiedTrees, totalTreesCount, 0, existingTree.State.NumberOfEntries, progressReport, $"Copying variable size tree '{treeName}'. Progress: 0/{existingTree.State.NumberOfEntries} entries.", treeName);
using (var existingTreeIterator = existingTree.Iterate(true))
{
if (existingTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
return(copiedTrees);
}
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
if (existingTree.IsLeafCompressionSupported)
{
txw.CreateTree(treeName, flags: TreeFlags.LeafsCompressed);
}
else
{
txw.CreateTree(treeName);
}
txw.Commit();
}
var copiedEntries = 0L;
do
{
var transactionSize = 0L;
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
var newTree = txw.ReadTree(treeName);
do
{
token.ThrowIfCancellationRequested();
var key = existingTreeIterator.CurrentKey;
if (existingTreeIterator.Current->Flags == TreeNodeFlags.MultiValuePageRef)
{
using (var multiTreeIterator = existingTree.MultiRead(key))
{
if (multiTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
continue;
}
do
{
token.ThrowIfCancellationRequested();
var multiValue = multiTreeIterator.CurrentKey;
newTree.MultiAdd(key, multiValue);
transactionSize += multiValue.Size;
} while (multiTreeIterator.MoveNext());
}
}
else if (existingTree.IsLeafCompressionSupported)
{
using (var read = existingTree.ReadDecompressed(key))
{
var value = read.Reader.AsStream();
newTree.Add(key, value);
transactionSize += value.Length;
}
}
else if (existingTree.State.Flags == (TreeFlags.FixedSizeTrees | TreeFlags.Streams))
{
var tag = existingTree.GetStreamTag(key);
using (var stream = existingTree.ReadStream(key))
{
if (tag != null)
{
Slice tagStr;
using (Slice.From(txw.Allocator, tag, out tagStr))
newTree.AddStream(key, stream, tagStr);
}
else
{
newTree.AddStream(key, stream);
}
transactionSize += stream.Length;
}
}
else
{
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
copiedEntries++;
var reportRate = existingTree.State.NumberOfEntries / 33 + 1;
if (copiedEntries % reportRate == 0)
{
Report(copiedTrees, totalTreesCount, copiedEntries, existingTree.State.NumberOfEntries, progressReport, $"Copying variable size tree '{treeName}'. Progress: {copiedEntries}/{existingTree.State.NumberOfEntries} entries.", treeName);
}
} while (transactionSize < compactedEnv.Options.MaxScratchBufferSize / 2 && existingTreeIterator.MoveNext());
txw.Commit();
}
if (copiedEntries == existingTree.State.NumberOfEntries)
{
copiedTrees++;
Report(copiedTrees, totalTreesCount, copiedEntries, existingTree.State.NumberOfEntries, progressReport, $"Finished copying variable size tree '{treeName}'. Progress: {copiedEntries}/{existingTree.State.NumberOfEntries} entries.", treeName);
}
compactedEnv.FlushLogToDataFile();
} while (existingTreeIterator.MoveNext());
}
return(copiedTrees);
}
private static long CopyTableTree(StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, Transaction txr, string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context, CancellationToken token)
{
// Load table
var tableTree = txr.ReadTree(treeName, RootObjectType.Table);
// Get the table schema
var schemaSize = tableTree.GetDataSize(TableSchema.SchemasSlice);
var schemaPtr = tableTree.DirectRead(TableSchema.SchemasSlice);
var schema = TableSchema.ReadFrom(txr.Allocator, schemaPtr, schemaSize);
// Load table into structure
var inputTable = txr.OpenTable(schema, treeName);
// The next three variables are used to know what our current
// progress is
var copiedEntries = 0;
// It is very important that these slices be allocated in the
// txr.Allocator, as the intermediate write transactions on
// the compacted environment will be destroyed between each
// loop.
var lastSlice = Slices.BeforeAllKeys;
long lastFixedIndex = 0L;
Report(copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Copying table tree '{treeName}'. Progress: {copiedEntries:#,#;;0}/{inputTable.NumberOfEntries:#,#;;0} entries.", treeName);
using (var txw = compactedEnv.WriteTransaction(context))
{
schema.Create(txw, treeName, Math.Max((ushort)inputTable.ActiveDataSmallSection.NumberOfPages, (ushort)((ushort.MaxValue + 1) / Constants.Storage.PageSize)));
txw.Commit(); // always create a table, even if it is empty
}
while (copiedEntries < inputTable.NumberOfEntries)
{
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
long transactionSize = 0L;
var outputTable = txw.OpenTable(schema, treeName);
if (schema.Key == null || schema.Key.IsGlobal)
{
// There is no primary key, or there is one that is global to multiple tables
// we require a table to have at least a single local index that we'll use
var variableSizeIndex = schema.Indexes.Values.FirstOrDefault(x => x.IsGlobal == false);
if (variableSizeIndex != null)
{
// We have a variable size index, use it
// In case we continue an existing compaction, skip to the next slice
var skip = 0;
// can't use SliceComparer.Compare here
if (lastSlice.Options != Slices.BeforeAllKeys.Options)
{
skip = 1;
}
var sp = Stopwatch.StartNew();
foreach (var tvr in inputTable.SeekForwardFrom(variableSizeIndex, lastSlice, skip))
{
// The table will take care of reconstructing indexes automatically
outputTable.Insert(ref tvr.Result.Reader);
copiedEntries++;
transactionSize += tvr.Result.Reader.Size;
ReportIfNeeded(sp, copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Copying table tree '{treeName}'. Progress: {copiedEntries:#,#;;0}/{inputTable.NumberOfEntries:#,#;;0} entries.", treeName);
// The transaction has surpassed the allowed
// size before a flush
if (lastSlice.Equals(tvr.Key) == false && transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2)
{
lastSlice = tvr.Key.Clone(txr.Allocator);
break;
}
}
}
else
{
// Use a fixed size index
var fixedSizeIndex = schema.FixedSizeIndexes.Values.FirstOrDefault(x => x.IsGlobal == false);
if (fixedSizeIndex == null)
{
throw new InvalidOperationException("Cannot compact table " + inputTable.Name + " because is has no local indexes, only global ones");
}
var sp = Stopwatch.StartNew();
foreach (var entry in inputTable.SeekForwardFrom(fixedSizeIndex, lastFixedIndex, lastFixedIndex > 0 ? 1 : 0))
{
token.ThrowIfCancellationRequested();
// The table will take care of reconstructing indexes automatically
outputTable.Insert(ref entry.Reader);
copiedEntries++;
transactionSize += entry.Reader.Size;
ReportIfNeeded(sp, copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Copying table tree '{treeName}'. Progress: {copiedEntries:#,#;;0}/{inputTable.NumberOfEntries:#,#;;0} entries.", treeName);
// The transaction has surpassed the allowed
// size before a flush
if (transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2)
{
lastFixedIndex = fixedSizeIndex.GetValue(ref entry.Reader);
break;
}
}
}
}
else
{
// The table has a primary key, inserts in that order are expected to be faster
foreach (var entry in inputTable.SeekByPrimaryKey(lastSlice, 0))
{
token.ThrowIfCancellationRequested();
// The table will take care of reconstructing indexes automatically
outputTable.Insert(ref entry.Reader);
copiedEntries++;
transactionSize += entry.Reader.Size;
// The transaction has surpassed the allowed
// size before a flush
if (transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2)
{
schema.Key.GetSlice(txr.Allocator, ref entry.Reader, out lastSlice);
break;
}
}
}
txw.Commit();
}
if (copiedEntries == inputTable.NumberOfEntries)
{
copiedTrees++;
Report(copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Finished copying table tree '{treeName}'. Progress: {copiedEntries:#,#;;0}/{inputTable.NumberOfEntries:#,#;;0} entries.", treeName);
}
compactedEnv.FlushLogToDataFile();
}
return(copiedTrees);
}
private static void CopyTrees(StorageEnvironment existingEnv, StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, CancellationToken token)
{
var context = new TransactionPersistentContext(true);
using (var txr = existingEnv.ReadTransaction(context))
using (var rootIterator = txr.LowLevelTransaction.RootObjects.Iterate(false))
{
if (rootIterator.Seek(Slices.BeforeAllKeys) == false)
{
return;
}
var globalTableIndexesToSkipCopying = new HashSet <string>();
do
{
var objectType = txr.GetRootObjectType(rootIterator.CurrentKey);
if (objectType != RootObjectType.Table)
{
continue;
}
var treeName = rootIterator.CurrentKey.ToString();
var tableTree = txr.ReadTree(treeName, RootObjectType.Table);
var schemaSize = tableTree.GetDataSize(TableSchema.SchemasSlice);
var schemaPtr = tableTree.DirectRead(TableSchema.SchemasSlice);
var schema = TableSchema.ReadFrom(txr.Allocator, schemaPtr, schemaSize);
if (schema.Key != null && schema.Key.IsGlobal)
{
globalTableIndexesToSkipCopying.Add(schema.Key.Name.ToString());
}
foreach (var index in schema.Indexes.Values)
{
if (index.IsGlobal)
{
globalTableIndexesToSkipCopying.Add(index.Name.ToString());
}
}
foreach (var index in schema.FixedSizeIndexes.Values)
{
if (index.IsGlobal)
{
globalTableIndexesToSkipCopying.Add(index.Name.ToString());
}
}
} while (rootIterator.MoveNext());
if (rootIterator.Seek(Slices.BeforeAllKeys) == false)
{
return;
}
// substract skipped items
var totalTreesCount = txr.LowLevelTransaction.RootObjects.State.NumberOfEntries - globalTableIndexesToSkipCopying.Count;
var copiedTrees = 0L;
do
{
token.ThrowIfCancellationRequested();
var treeName = rootIterator.CurrentKey.ToString();
if (globalTableIndexesToSkipCopying.Contains(treeName))
{
continue;
}
var currentKey = rootIterator.CurrentKey.Clone(txr.Allocator);
var objectType = txr.GetRootObjectType(currentKey);
switch (objectType)
{
case RootObjectType.None:
break;
case RootObjectType.VariableSizeTree:
copiedTrees = CopyVariableSizeTree(compactedEnv, progressReport, txr, treeName, copiedTrees, totalTreesCount, context, token);
break;
case RootObjectType.EmbeddedFixedSizeTree:
case RootObjectType.FixedSizeTree:
if (FreeSpaceHandling.IsFreeSpaceTreeName(treeName))
{
copiedTrees++;// we don't copy the fixed size tree
continue;
}
if (NewPageAllocator.AllocationStorageName == treeName)
{
copiedTrees++;
continue; // we don't copy the allocator storage
}
copiedTrees = CopyFixedSizeTrees(compactedEnv, progressReport, txr, rootIterator, treeName, copiedTrees, totalTreesCount, context, token);
break;
case RootObjectType.Table:
copiedTrees = CopyTableTree(compactedEnv, progressReport, txr, treeName, copiedTrees, totalTreesCount, context, token);
break;
default:
throw new ArgumentOutOfRangeException("Unknown " + objectType);
}
} while (rootIterator.MoveNext());
}
}
private static long CopyFixedSizeTrees(StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, Transaction txr,
TreeIterator rootIterator, string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context, CancellationToken token)
{
var treeNameSlice = rootIterator.CurrentKey.Clone(txr.Allocator);
var header = (FixedSizeTreeHeader.Embedded *)txr.LowLevelTransaction.RootObjects.DirectRead(treeNameSlice);
var fst = txr.FixedTreeFor(treeNameSlice, header->ValueSize);
Report(copiedTrees, totalTreesCount, 0, fst.NumberOfEntries, progressReport, $"Copying fixed size tree '{treeName}'. Progress: 0/{fst.NumberOfEntries} entries.", treeName);
using (var it = fst.Iterate())
{
var copiedEntries = 0L;
if (it.Seek(Int64.MinValue) == false)
{
return(copiedTrees);
}
do
{
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
var snd = txw.FixedTreeFor(treeNameSlice, header->ValueSize);
var transactionSize = 0L;
do
{
token.ThrowIfCancellationRequested();
Slice val;
using (it.Value(out val))
snd.Add(it.CurrentKey, val);
transactionSize += fst.ValueSize + sizeof(long);
copiedEntries++;
var reportRate = fst.NumberOfEntries / 33 + 1;
if (copiedEntries % reportRate == 0)
{
Report(copiedTrees, totalTreesCount, copiedEntries, fst.NumberOfEntries, progressReport, $"Copying fixed size tree '{treeName}'. Progress: {copiedEntries}/{fst.NumberOfEntries} entries.", treeName);
}
} while (transactionSize < compactedEnv.Options.MaxScratchBufferSize / 2 && it.MoveNext());
txw.Commit();
}
if (fst.NumberOfEntries == copiedEntries)
{
copiedTrees++;
Report(copiedTrees, totalTreesCount, copiedEntries, fst.NumberOfEntries, progressReport, $"Finished copying fixed size tree '{treeName}'. Progress: {copiedEntries}/{fst.NumberOfEntries} entries.", treeName);
}
compactedEnv.FlushLogToDataFile();
} while (it.MoveNext());
}
return(copiedTrees);
}
private static void CopyFixedSizeTree(FixedSizeTree fst, Func <Transaction, FixedSizeTree> createDestinationTree, StorageEnvironment compactedEnv, TransactionPersistentContext context, Action <long> onEntriesCopiedProgress, Action onAllEntriesCopied, CancellationToken token)
{
using (var it = fst.Iterate())
{
var copiedEntries = 0L;
if (it.Seek(Int64.MinValue) == false)
{
return;
}
do
{
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
var snd = createDestinationTree(txw);
var transactionSize = 0L;
do
{
token.ThrowIfCancellationRequested();
using (it.Value(out var val))
snd.Add(it.CurrentKey, val);
transactionSize += fst.ValueSize + sizeof(long);
copiedEntries++;
var reportRate = fst.NumberOfEntries / 33 + 1;
if (copiedEntries % reportRate == 0)
{
onEntriesCopiedProgress(copiedEntries);
}
} while (transactionSize < compactedEnv.Options.MaxScratchBufferSize / 2 && it.MoveNext());
txw.Commit();
}
compactedEnv.FlushLogToDataFile();
if (fst.NumberOfEntries == copiedEntries)
{
onAllEntriesCopied();
}
} while (it.MoveNext());
}
}
private static long CopyVariableSizeTree(StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, Transaction txr, string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context, CancellationToken token)
{
var existingTree = txr.ReadTree(treeName);
Report(copiedTrees, totalTreesCount, 0, existingTree.State.NumberOfEntries, progressReport, $"Copying variable size tree '{treeName}'. Progress: 0/{existingTree.State.NumberOfEntries} entries.", treeName);
using (var existingTreeIterator = existingTree.Iterate(true))
{
if (existingTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
return(copiedTrees);
}
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
if (existingTree.IsLeafCompressionSupported)
{
txw.CreateTree(treeName, flags: TreeFlags.LeafsCompressed);
}
else
{
txw.CreateTree(treeName);
}
txw.Commit();
}
var copiedEntries = 0L;
do
{
var transactionSize = 0L;
token.ThrowIfCancellationRequested();
var txw = compactedEnv.WriteTransaction(context);
try
{
var newTree = txw.ReadTree(treeName);
do
{
token.ThrowIfCancellationRequested();
var key = existingTreeIterator.CurrentKey;
if (existingTreeIterator.Current->Flags == TreeNodeFlags.MultiValuePageRef)
{
using (var multiTreeIterator = existingTree.MultiRead(key))
{
if (multiTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
continue;
}
do
{
token.ThrowIfCancellationRequested();
var multiValue = multiTreeIterator.CurrentKey;
newTree.MultiAdd(key, multiValue);
transactionSize += multiValue.Size;
} while (multiTreeIterator.MoveNext());
}
}
else if (existingTree.IsLeafCompressionSupported)
{
using (var read = existingTree.ReadDecompressed(key))
{
var value = read.Reader.AsStream();
newTree.Add(key, value);
transactionSize += value.Length;
}
}
else if (existingTree.State.Flags == (TreeFlags.FixedSizeTrees | TreeFlags.Streams))
{
var tag = existingTree.GetStreamTag(key);
using (var stream = existingTree.ReadStream(key))
{
if (tag != null)
{
Slice tagStr;
using (Slice.From(txw.Allocator, tag, out tagStr))
newTree.AddStream(key, stream, tagStr);
}
else
{
newTree.AddStream(key, stream);
}
transactionSize += stream.Length;
}
}
else if (existingTree.State.Flags == TreeFlags.FixedSizeTrees)
{
var reader = existingTree.GetValueReaderFromHeader(existingTreeIterator.Current);
if (reader.Length >= sizeof(FixedSizeTreeHeader.Embedded))
{
var header = (FixedSizeTreeHeader.Embedded *)reader.Base;
if (header->RootObjectType == RootObjectType.FixedSizeTree || header->RootObjectType == RootObjectType.EmbeddedFixedSizeTree)
{
// CopyFixedSizeTree will open dedicated write transaction to copy fixed size tree
txw.Commit();
txw.Dispose();
txw = null;
var fixedSizeTreeName = key;
var fst = existingTree.FixedTreeFor(fixedSizeTreeName, (byte)header->ValueSize);
var currentCopiedTrees = copiedTrees;
var currentCopiedEntries = copiedEntries;
CopyFixedSizeTree(fst, tx =>
{
var treeInCompactedEnv = tx.ReadTree(treeName);
return(treeInCompactedEnv.FixedTreeFor(fixedSizeTreeName, (byte)header->ValueSize));
}, compactedEnv, context, copiedFstEntries =>
{
Report(currentCopiedTrees, totalTreesCount, currentCopiedEntries, existingTree.State.NumberOfEntries, progressReport,
$"Copying fixed size tree '{fixedSizeTreeName}' inside '{treeName}' tree. Progress: {copiedFstEntries}/{fst.NumberOfEntries} entries.",
treeName);
}, () =>
{
Report(currentCopiedTrees, totalTreesCount, currentCopiedEntries, existingTree.State.NumberOfEntries, progressReport,
$"Finished copying fixed size tree '{fixedSizeTreeName}' inside '{treeName}' tree. {fst.NumberOfEntries} entries copied.",
treeName);
}, token);
IncrementNumberOfCopiedEntries();
break; // let's open new transaction after copying fixed size tree
}
}
// if the entry wasn't recognized as fixed size tree then let's store it as regular value
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
else
{
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
IncrementNumberOfCopiedEntries();
void IncrementNumberOfCopiedEntries()
{
copiedEntries++;
var reportRate = existingTree.State.NumberOfEntries / 33 + 1;
if (copiedEntries % reportRate == 0)
{
Report(copiedTrees, totalTreesCount, copiedEntries, existingTree.State.NumberOfEntries, progressReport,
$"Copying variable size tree '{treeName}'. Progress: {copiedEntries}/{existingTree.State.NumberOfEntries} entries.", treeName);
}
}
} while (transactionSize < compactedEnv.Options.MaxScratchBufferSize / 2 && existingTreeIterator.MoveNext());
txw?.Commit();
}
finally
{
txw?.Dispose();
}
if (copiedEntries == existingTree.State.NumberOfEntries)
{
copiedTrees++;
Report(copiedTrees, totalTreesCount, copiedEntries, existingTree.State.NumberOfEntries, progressReport, $"Finished copying variable size tree '{treeName}'. Progress: {copiedEntries}/{existingTree.State.NumberOfEntries} entries.", treeName);
}
compactedEnv.FlushLogToDataFile();
} while (existingTreeIterator.MoveNext());
}
return(copiedTrees);
}
public LowLevelTransaction(StorageEnvironment env, long id, TransactionPersistentContext transactionPersistentContext, TransactionFlags flags, IFreeSpaceHandling freeSpaceHandling, ByteStringContext context = null)
{
TxStartTime = DateTime.UtcNow;
if (flags == TransactionFlags.ReadWrite)
{
env.Options.AssertNoCatastrophicFailure();
}
DataPager = env.Options.DataPager;
_env = env;
_journal = env.Journal;
_id = id;
_freeSpaceHandling = freeSpaceHandling;
_allocator = context ?? new ByteStringContext(SharedMultipleUseFlag.None);
_disposeAllocator = context == null;
_pagerStates = new HashSet <PagerState>(ReferenceEqualityComparer <PagerState> .Default);
PersistentContext = transactionPersistentContext;
Flags = flags;
var scratchPagerStates = env.ScratchBufferPool.GetPagerStatesOfAllScratches();
foreach (var scratchPagerState in scratchPagerStates.Values)
{
scratchPagerState.AddRef();
_pagerStates.Add(scratchPagerState);
}
_pageLocator = transactionPersistentContext.AllocatePageLocator(this);
if (flags != TransactionFlags.ReadWrite)
{
// for read transactions, we need to keep the pager state frozen
// for write transactions, we can use the current one (which == null)
_scratchPagerStates = scratchPagerStates;
_state = env.State.Clone();
InitializeRoots();
JournalSnapshots = _journal.GetSnapshots();
return;
}
EnsureNoDuplicateTransactionId(id);
// we keep this copy to make sure that if we use async commit, we have a stable copy of the jounrals
// as they were at the time we started the original transaction, this is required because async commit
// may modify the list of files we have available
JournalFiles = _journal.Files;
foreach (var journalFile in JournalFiles)
{
journalFile.AddRef();
}
_env.WriteTransactionPool.Reset();
_dirtyOverflowPages = _env.WriteTransactionPool.DirtyOverflowPagesPool;
_scratchPagesTable = _env.WriteTransactionPool.ScratchPagesInUse;
_dirtyPages = _env.WriteTransactionPool.DirtyPagesPool;
_freedPages = new HashSet <long>(NumericEqualityComparer.BoxedInstanceInt64);
_unusedScratchPages = new List <PageFromScratchBuffer>();
_transactionPages = new HashSet <PageFromScratchBuffer>(PageFromScratchBufferEqualityComparer.Instance);
_pagesToFreeOnCommit = new Stack <long>();
_state = env.State.Clone();
InitializeRoots();
InitTransactionHeader();
}
private static long CopyVariableSizeTree(StorageEnvironment compactedEnv, Action <CompactionProgress> progressReport, Transaction txr,
string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context)
{
var existingTree = txr.ReadTree(treeName);
Report(RootObjectType.VariableSizeTree, treeName, copiedTrees, totalTreesCount, 0, existingTree.State.NumberOfEntries, progressReport);
using (var existingTreeIterator = existingTree.Iterate(true))
{
if (existingTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
return(copiedTrees);
}
using (var txw = compactedEnv.WriteTransaction(context))
{
txw.CreateTree(treeName);
txw.Commit();
}
var copiedEntries = 0L;
do
{
var transactionSize = 0L;
using (var txw = compactedEnv.WriteTransaction(context))
{
var newTree = txw.ReadTree(treeName);
do
{
var key = existingTreeIterator.CurrentKey;
if (existingTreeIterator.Current->Flags == TreeNodeFlags.MultiValuePageRef)
{
using (var multiTreeIterator = existingTree.MultiRead(key))
{
if (multiTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
continue;
}
do
{
var multiValue = multiTreeIterator.CurrentKey;
newTree.MultiAdd(key, multiValue);
transactionSize += multiValue.Size;
} while (multiTreeIterator.MoveNext());
}
}
else
{
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
copiedEntries++;
} while (transactionSize < compactedEnv.Options.MaxScratchBufferSize / 2 && existingTreeIterator.MoveNext());
txw.Commit();
}
if (copiedEntries == existingTree.State.NumberOfEntries)
{
copiedTrees++;
}
Report(RootObjectType.VariableSizeTree, treeName, copiedTrees, totalTreesCount, copiedEntries, existingTree.State.NumberOfEntries, progressReport);
compactedEnv.FlushLogToDataFile();
} while (existingTreeIterator.MoveNext());
}
return(copiedTrees);
}
private static long CopyTableTree(StorageEnvironment compactedEnv, Action <CompactionProgress> progressReport, Transaction txr,
string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context)
{
// Load table
var tableTree = txr.ReadTree(treeName, RootObjectType.Table);
// Get the table schema
var schemaSize = tableTree.GetDataSize(TableSchema.SchemasSlice);
var schemaPtr = tableTree.DirectRead(TableSchema.SchemasSlice);
var schema = TableSchema.ReadFrom(txr.Allocator, schemaPtr, schemaSize);
// Load table into structure
var inputTable = txr.OpenTable(schema, treeName);
// The next three variables are used to know what our current
// progress is
var copiedEntries = 0;
// It is very important that these slices be allocated in the
// txr.Allocator, as the intermediate write transactions on
// the compacted environment will be destroyed between each
// loop.
var lastSlice = Slices.BeforeAllKeys;
long lastFixedIndex = 0L;
Report(RootObjectType.Table, treeName, copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport);
while (copiedEntries < inputTable.NumberOfEntries)
{
using (var txw = compactedEnv.WriteTransaction(context))
{
long transactionSize = 0L;
schema.Create(txw, treeName, Math.Max((ushort)inputTable.ActiveDataSmallSection.NumberOfPages, ushort.MaxValue));
var outputTable = txw.OpenTable(schema, treeName);
if (schema.Key == null)
{
// There is no primary key, however, there must be at least one index
if (schema.Indexes.Count > 0)
{
// We have a variable size index, use it
var index = schema.Indexes.First().Value;
foreach (var result in inputTable.SeekForwardFrom(index, lastSlice))
{
foreach (var entry in result.Results)
{
// The table will take care of reconstructing indexes automatically
outputTable.Insert(entry);
copiedEntries++;
transactionSize += entry.Size;
}
// The transaction has surpassed the allowed
// size before a flush
if (transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2)
{
lastSlice = result.Key;
break;
}
}
}
else
{
// Use a fixed size index
var index = schema.FixedSizeIndexes.First().Value;
foreach (var entry in inputTable.SeekForwardFrom(index, lastFixedIndex))
{
// The table will take care of reconstructing indexes automatically
outputTable.Insert(entry);
copiedEntries++;
transactionSize += entry.Size;
// The transaction has surpassed the allowed
// size before a flush
if (transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2)
{
lastFixedIndex = index.GetValue(entry);
break;
}
}
}
}
else
{
// The table has a primary key, inserts in that order are expected to be faster
foreach (var entry in inputTable.SeekByPrimaryKey(lastSlice))
{
// The table will take care of reconstructing indexes automatically
outputTable.Insert(entry);
copiedEntries++;
transactionSize += entry.Size;
// The transaction has surpassed the allowed
// size before a flush
if (transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2)
{
schema.Key.GetSlice(txr.Allocator, entry, out lastSlice);
break;
}
}
}
txw.Commit();
}
if (copiedEntries == inputTable.NumberOfEntries)
{
copiedTrees++;
}
Report(RootObjectType.Table, treeName, copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport);
compactedEnv.FlushLogToDataFile();
}
return(copiedTrees);
}
public LazyTransactionBuffer(StorageEnvironmentOptions options)
{
_lazyTransactionPager = options.CreateScratchPager("lazy-transactions.buffer", options.InitialFileSize ?? options.InitialLogFileSize);
_transactionPersistentContext = new TransactionPersistentContext(true);
}
private static void Backup(
StorageEnvironment env, CompressionLevel compression, Action <string> infoNotify,
Action backupStarted, AbstractPager dataPager, ZipArchive package, string basePath, DataCopier copier)
{
var usedJournals = new List <JournalFile>();
long lastWrittenLogPage = -1;
long lastWrittenLogFile = -1;
LowLevelTransaction txr = null;
var backupSuccess = false;
try
{
long allocatedPages;
var writePesistentContext = new TransactionPersistentContext(true);
var readPesistentContext = new TransactionPersistentContext(true);
using (var txw = env.NewLowLevelTransaction(writePesistentContext, TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
{
txr = env.NewLowLevelTransaction(readPesistentContext, TransactionFlags.Read); // now have snapshot view
allocatedPages = dataPager.NumberOfAllocatedPages;
Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);
infoNotify("Voron copy headers for " + basePath);
VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options, basePath);
// journal files snapshot
var files = env.Journal.Files; // thread safety copy
JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1;
journalNum <= journalInfo.CurrentJournal;
journalNum++)
{
var journalFile = files.FirstOrDefault(x => x.Number == journalNum);
// first check journal files currently being in use
if (journalFile == null)
{
long journalSize;
using (var pager = env.Options.OpenJournalPager(journalNum))
{
journalSize = Bits.NextPowerOf2(pager.NumberOfAllocatedPages * Constants.Storage.PageSize);
}
journalFile = new JournalFile(env, env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
}
journalFile.AddRef();
usedJournals.Add(journalFile);
}
if (env.Journal.CurrentFile != null)
{
lastWrittenLogFile = env.Journal.CurrentFile.Number;
lastWrittenLogPage = env.Journal.CurrentFile.WritePosIn4KbPosition - 1;
}
// txw.Commit(); intentionally not committing
}
backupStarted?.Invoke();
// data file backup
var dataPart = package.CreateEntry(Path.Combine(basePath, Constants.DatabaseFilename), compression);
Debug.Assert(dataPart != null);
if (allocatedPages > 0) //only true if dataPager is still empty at backup start
{
using (var dataStream = dataPart.Open())
{
// now can copy everything else
copier.ToStream(dataPager, 0, allocatedPages, dataStream);
}
}
try
{
long lastBackedupJournal = 0;
foreach (var journalFile in usedJournals)
{
var entryName = StorageEnvironmentOptions.JournalName(journalFile.Number);
var journalPart = package.CreateEntry(Path.Combine(basePath, entryName), compression);
Debug.Assert(journalPart != null);
long pagesToCopy = journalFile.JournalWriter.NumberOfAllocated4Kb;
if (journalFile.Number == lastWrittenLogFile)
{
pagesToCopy = lastWrittenLogPage + 1;
}
using (var stream = journalPart.Open())
{
copier.ToStream(env, journalFile, 0, pagesToCopy, stream);
infoNotify(string.Format("Voron copy journal file {0}", entryName));
}
lastBackedupJournal = journalFile.Number;
}
if (env.Options.IncrementalBackupEnabled)
{
env.HeaderAccessor.Modify(header =>
{
header->IncrementalBackup.LastBackedUpJournal = lastBackedupJournal;
//since we backed-up everything, no need to start next incremental backup from the middle
header->IncrementalBackup.LastBackedUpJournalPage = -1;
});
}
backupSuccess = true;
}
catch (Exception)
{
backupSuccess = false;
throw;
}
finally
{
var lastSyncedJournal = env.HeaderAccessor.Get(header => header->Journal).LastSyncedJournal;
foreach (var journalFile in usedJournals)
{
if (backupSuccess) // if backup succeeded we can remove journals
{
if (journalFile.Number < lastWrittenLogFile && // prevent deletion of the current journal and journals with a greater number
journalFile.Number < lastSyncedJournal) // prevent deletion of journals that aren't synced with the data file
{
journalFile.DeleteOnClose = true;
}
}
journalFile.Release();
}
}
}
finally
{
txr?.Dispose();
}
}
private void OpenWrite()
{
var writePersistentContext = new TransactionPersistentContext(true);
Write = new Transaction(_env.NewLowLevelTransaction(writePersistentContext, TransactionFlags.ReadWrite));
}
public void OpenRead()
{
var readPersistentContext = new TransactionPersistentContext(true);
Read = new Transaction(_env.NewLowLevelTransaction(readPersistentContext, TransactionFlags.Read));
}
private static long Incremental_Backup(StorageEnvironment env, CompressionLevel compression, Action <string> infoNotify,
Action backupStarted, ZipArchive package, string basePath, DataCopier copier)
{
long numberOfBackedUpPages = 0;
long lastWrittenLogFile = -1;
long lastWrittenLog4kb = -1;
bool backupSuccess = true;
IncrementalBackupInfo backupInfo;
JournalInfo journalInfo;
var transactionPersistentContext = new TransactionPersistentContext(true);
using (var txw = env.NewLowLevelTransaction(transactionPersistentContext, TransactionFlags.ReadWrite))
{
backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup);
journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
if (env.Journal.CurrentFile != null)
{
lastWrittenLogFile = env.Journal.CurrentFile.Number;
lastWrittenLog4kb = env.Journal.CurrentFile.WritePosIn4KbPosition;
}
// txw.Commit(); intentionally not committing
}
using (env.NewLowLevelTransaction(transactionPersistentContext, TransactionFlags.Read))
{
backupStarted?.Invoke(); // we let call know that we have started the backup
var usedJournals = new List <JournalFile>();
try
{
long lastBackedUpPage = -1;
long lastBackedUpFile = -1;
var firstJournalToBackup = backupInfo.LastBackedUpJournal;
if (firstJournalToBackup == -1)
{
firstJournalToBackup = 0; // first time that we do incremental backup
}
for (var journalNum = firstJournalToBackup;
journalNum <= backupInfo.LastCreatedJournal;
journalNum++)
{
var num = journalNum;
var journalFile = GetJournalFile(env, journalNum, backupInfo, journalInfo);
journalFile.AddRef();
usedJournals.Add(journalFile);
var startBackupAt = 0L;
long numberOf4KbsToCopy = journalFile.JournalWriter.NumberOfAllocated4Kb;
if (journalFile.Number == backupInfo.LastBackedUpJournal)
{
startBackupAt = backupInfo.LastBackedUpJournalPage + 1;
numberOf4KbsToCopy -= startBackupAt;
}
if (startBackupAt >= journalFile.JournalWriter.NumberOfAllocated4Kb) // nothing to do here
{
continue;
}
var part =
package.CreateEntry(
Path.Combine(basePath, StorageEnvironmentOptions.JournalName(journalNum))
, compression);
Debug.Assert(part != null);
if (journalFile.Number == lastWrittenLogFile)
{
numberOf4KbsToCopy -= (journalFile.JournalWriter.NumberOfAllocated4Kb - lastWrittenLog4kb);
}
using (var stream = part.Open())
{
copier.ToStream(env, journalFile, startBackupAt, numberOf4KbsToCopy, stream);
infoNotify(string.Format("Voron Incr copy journal number {0}", num));
}
lastBackedUpFile = journalFile.Number;
if (journalFile.Number == backupInfo.LastCreatedJournal)
{
lastBackedUpPage = startBackupAt + numberOf4KbsToCopy - 1;
// we used all of this file, so the next backup should start in the next file
if (lastBackedUpPage == (journalFile.JournalWriter.NumberOfAllocated4Kb - 1))
{
lastBackedUpPage = -1;
lastBackedUpFile++;
}
}
numberOfBackedUpPages += numberOf4KbsToCopy;
}
env.HeaderAccessor.Modify(header =>
{
header->IncrementalBackup.LastBackedUpJournal = lastBackedUpFile;
header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage;
});
}
catch (Exception)
{
backupSuccess = false;
throw;
}
finally
{
var lastSyncedJournal = env.HeaderAccessor.Get(header => header->Journal).LastSyncedJournal;
foreach (var jrnl in usedJournals)
{
if (backupSuccess) // if backup succeeded we can remove journals
{
if (jrnl.Number < lastWrittenLogFile &&
// prevent deletion of the current journal and journals with a greater number
jrnl.Number < lastSyncedJournal)
// prevent deletion of journals that aren't synced with the data file
{
jrnl.DeleteOnClose = true;
}
}
jrnl.Release();
}
}
infoNotify(string.Format("Voron Incr Backup total {0} pages", numberOfBackedUpPages));
}
return(numberOfBackedUpPages);
}
private static long CopyFixedSizeTrees(StorageEnvironment compactedEnv, Action <CompactionProgress> progressReport, Transaction txr,
TreeIterator rootIterator, string treeName, long copiedTrees, long totalTreesCount, RootObjectType type, TransactionPersistentContext context)
{
var fst = txr.FixedTreeFor(rootIterator.CurrentKey.Clone(txr.Allocator), 0);
Report(type, treeName, copiedTrees, totalTreesCount, 0, fst.NumberOfEntries, progressReport);
using (var it = fst.Iterate())
{
var copiedEntries = 0L;
if (it.Seek(Int64.MinValue) == false)
{
return(copiedTrees);
}
do
{
using (var txw = compactedEnv.WriteTransaction(context))
{
var snd = txw.FixedTreeFor(rootIterator.CurrentKey.Clone(txr.Allocator));
var transactionSize = 0L;
do
{
Slice val;
using (it.Value(out val))
snd.Add(it.CurrentKey, val);
transactionSize += fst.ValueSize + sizeof(long);
copiedEntries++;
} while (transactionSize < compactedEnv.Options.MaxScratchBufferSize / 2 && it.MoveNext());
txw.Commit();
}
if (fst.NumberOfEntries == copiedEntries)
{
copiedTrees++;
}
Report(type, treeName, copiedTrees, totalTreesCount, copiedEntries, fst.NumberOfEntries, progressReport);
compactedEnv.FlushLogToDataFile();
} while (it.MoveNext());
}
return(copiedTrees);
}
