private static void DeleteTombstoneIfNeeded(DocumentsOperationContext context, CollectionName collectionName, byte *lowerId, int lowerSize)
{
var tombstoneTable = context.Transaction.InnerTransaction.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones));
using (Slice.External(context.Allocator, lowerId, lowerSize, out Slice id))
{
if (tombstoneTable.ReadByKey(id, out var reader) == false)
{
return;
}
if (tombstoneTable.IsOwned(reader.Id))
{
tombstoneTable.Delete(reader.Id);
return;
}
// this is using a different collection, so we need to handle that.
collectionName = new CollectionName(TableValueToId(context, (int)TombstoneTable.Collection, ref reader));
tombstoneTable = context.Transaction.InnerTransaction.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones));
tombstoneTable.Delete(reader.Id);
}
}
private static void ThrowInvalidCollectionNameChange(string id, CollectionName oldCollectionName, CollectionName collectionName)
{
throw new InvalidOperationException(
$"Changing '{id}' from '{oldCollectionName.Name}' to '{collectionName.Name}' via update is not supported.{Environment.NewLine}" +
$"Delete it and recreate the document {id}.");
}
protected bool Equals(CollectionName other)
{
return(string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase));
}
public void AddConflict(
DocumentsOperationContext context,
string id,
long lastModifiedTicks,
BlittableJsonReaderObject incomingDoc,
string incomingChangeVector,
string incomingTombstoneCollection,
DocumentFlags flags,
NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Adding conflict to {id} (Incoming change vector {incomingChangeVector})");
}
var tx = context.Transaction.InnerTransaction;
var conflictsTable = tx.OpenTable(ConflictsSchema, ConflictsSlice);
var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler;
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
CollectionName collectionName;
// ReSharper disable once ArgumentsStyleLiteral
var existing = _documentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false);
if (existing.Document != null)
{
var existingDoc = existing.Document;
if (fromSmuggler == false)
{
using (Slice.From(context.Allocator, existingDoc.ChangeVector, out Slice cv))
using (DocumentIdWorker.GetStringPreserveCase(context, CollectionName.GetLazyCollectionNameFrom(context, existingDoc.Data), out Slice collectionSlice))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv);
tvb.Add(idPtr);
tvb.Add(existingDoc.Data.BasePointer, existingDoc.Data.Size);
tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag()));
tvb.Add(collectionSlice);
tvb.Add(existingDoc.LastModified.Ticks);
tvb.Add((int)existingDoc.Flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
}
// we delete the data directly, without generating a tombstone, because we have a
// conflict instead
_documentsStorage.EnsureLastEtagIsPersisted(context, existingDoc.Etag);
collectionName = _documentsStorage.ExtractCollectionName(context, existingDoc.Data);
//make sure that the relevant collection tree exists
var table = tx.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
table.Delete(existingDoc.StorageId);
}
else if (existing.Tombstone != null)
{
var existingTombstone = existing.Tombstone;
if (fromSmuggler == false)
{
using (Slice.From(context.Allocator, existingTombstone.ChangeVector, out var cv))
using (DocumentIdWorker.GetStringPreserveCase(context, existingTombstone.Collection, out Slice collectionSlice))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv);
tvb.Add(idPtr);
tvb.Add(null, 0);
tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag()));
tvb.Add(collectionSlice);
tvb.Add(existingTombstone.LastModified.Ticks);
tvb.Add((int)existingTombstone.Flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
}
// we delete the data directly, without generating a tombstone, because we have a
// conflict instead
_documentsStorage.EnsureLastEtagIsPersisted(context, existingTombstone.Etag);
collectionName = _documentsStorage.GetCollection(existingTombstone.Collection, throwIfDoesNotExist: true);
var table = tx.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones));
table.Delete(existingTombstone.StorageId);
}
else // has existing conflicts
{
collectionName = _documentsStorage.ExtractCollectionName(context, incomingDoc);
using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice))
{
var conflicts = GetConflictsFor(context, prefixSlice);
foreach (var conflict in conflicts)
{
var conflictStatus = ChangeVectorUtils.GetConflictStatus(incomingChangeVector, conflict.ChangeVector);
switch (conflictStatus)
{
case ConflictStatus.Update:
DeleteConflictsFor(context, conflict.ChangeVector); // delete this, it has been subsumed
break;
case ConflictStatus.Conflict:
if (fromSmuggler &&
DocumentCompare.IsEqualTo(conflict.Doc, incomingDoc, false) == DocumentCompareResult.Equal)
{
return; // we already have a conflict with equal content, no need to create another one
}
break; // we'll add this conflict if no one else also includes it
case ConflictStatus.AlreadyMerged:
return; // we already have a conflict that includes this version
default:
throw new ArgumentOutOfRangeException("Invalid conflict status " + conflictStatus);
}
}
}
}
var etag = _documentsStorage.GenerateNextEtag();
if (context.LastDatabaseChangeVector == null)
{
context.LastDatabaseChangeVector = GetDatabaseChangeVector(context);
}
var result = ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentDatabase.DbBase64Id, etag, context.LastDatabaseChangeVector);
if (result.IsValid)
{
context.LastDatabaseChangeVector = result.ChangeVector;
}
byte * doc = null;
var docSize = 0;
string collection;
if (incomingDoc != null) // can be null if it is a tombstone
{
doc = incomingDoc.BasePointer;
docSize = incomingDoc.Size;
collection = CollectionName.GetLazyCollectionNameFrom(context, incomingDoc);
}
else
{
collection = incomingTombstoneCollection;
}
using (Slice.From(context.Allocator, incomingChangeVector, out var cv))
using (DocumentIdWorker.GetStringPreserveCase(context, collection, out Slice collectionSlice))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv);
tvb.Add(idPtr);
tvb.Add(doc, docSize);
tvb.Add(Bits.SwapBytes(etag));
tvb.Add(collectionSlice);
tvb.Add(lastModifiedTicks);
tvb.Add((int)flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = incomingChangeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Conflict,
});
}
}
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result)
{
if (_isCompactionInProgress)
{
throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress.");
}
result.AddMessage($"Started database compaction for {_database}");
onProgress?.Invoke(result.Progress);
_isCompactionInProgress = true;
bool done = false;
string compactDirectory = null;
string tmpDirectory = null;
string compactTempDirectory = null;
byte[] encryptionKey = null;
try
{
var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database);
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
DatabaseRecord databaseRecord = documentDatabase.ReadDatabaseRecord();
// save the key before unloading the database (it is zeroed when disposing DocumentDatabase).
if (documentDatabase.MasterKey != null)
{
encryptionKey = documentDatabase.MasterKey.ToArray();
}
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database, "it is being compacted"))
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications
{
DisableIoMetrics = true
},
new CatastrophicFailureNotification((endId, path, exception, stacktrace) => throw new InvalidOperationException($"Failed to compact database {_database} ({path}), StackTrace='{stacktrace}'", exception))))
{
InitializeOptions(src, configuration, documentDatabase, encryptionKey);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(src, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
var basePath = configuration.Core.DataDirectory.FullPath;
compactDirectory = basePath + "-compacting";
tmpDirectory = basePath + "-old";
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(compactDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
configuration.Core.DataDirectory = new PathSetting(compactDirectory);
if (configuration.Storage.TempPath != null)
{
compactTempDirectory = configuration.Storage.TempPath.FullPath + "-temp-compacting";
EnsureDirectoriesPermission(compactTempDirectory);
IOExtensions.DeleteDirectory(compactTempDirectory);
configuration.Storage.TempPath = new PathSetting(compactTempDirectory);
}
var revisionsPrefix = CollectionName.GetTablePrefix(CollectionTableType.Revisions);
var compressedCollectionsTableNames = databaseRecord.DocumentsCompression?.Collections
.Select(name => new CollectionName(name).GetTableName(CollectionTableType.Documents))
.ToHashSet(StringComparer.OrdinalIgnoreCase);
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications
{
DisableIoMetrics = true
},
new CatastrophicFailureNotification((envId, path, exception, stacktrace) => throw new InvalidOperationException($"Failed to compact database {_database} ({path}). StackTrace='{stacktrace}'", exception))))
{
InitializeOptions(dst, configuration, documentDatabase, encryptionKey);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(dst, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
result.Progress.TreeProgress = progressReport.TreeProgress;
result.Progress.TreeTotal = progressReport.TreeTotal;
result.Progress.TreeName = progressReport.TreeName;
result.Progress.GlobalProgress = progressReport.GlobalProgress;
result.Progress.GlobalTotal = progressReport.GlobalTotal;
result.AddMessage(progressReport.Message);
onProgress?.Invoke(result.Progress);
}, (name, schema) =>
{
bool isRevision = name.StartsWith(revisionsPrefix, StringComparison.OrdinalIgnoreCase);
schema.Compressed =
(isRevision && databaseRecord.DocumentsCompression?.CompressRevisions == true) ||
compressedCollectionsTableNames?.Contains(name) == true;
}, _token);
}
result.TreeName = null;
_token.ThrowIfCancellationRequested();
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
SwitchDatabaseDirectories(basePath, tmpDirectory, compactDirectory);
done = true;
}
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(compactDirectory);
if (done)
{
IOExtensions.DeleteDirectory(tmpDirectory);
if (compactTempDirectory != null)
{
IOExtensions.DeleteDirectory(compactTempDirectory);
}
}
_isCompactionInProgress = false;
if (encryptionKey != null)
{
Sodium.ZeroBuffer(encryptionKey);
}
}
}
private static bool IsHiLoDocument(Document document)
{
var collection = CollectionName.GetCollectionName(document.Data);
return(CollectionName.IsHiLoCollection(collection));
}
private static void DeleteTombstoneIfNeeded(DocumentsOperationContext context, CollectionName collectionName, byte *lowerId, int lowerSize)
{
var tombstoneTable = context.Transaction.InnerTransaction.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones));
using (Slice.External(context.Allocator, lowerId, lowerSize, out Slice id))
{
if (tombstoneTable.ReadByKey(id, out var reader) == false)
{
return;
}
if (tombstoneTable.IsOwned(reader.Id))
{
tombstoneTable.Delete(reader.Id);
}
}
}
private static unsafe bool IsSystemDocument(Document document)
{
return(CollectionName.IsSystemDocument(document.Id.Buffer, document.Id.Length, out var _));
}
