public void ValidateAtomicGuard(string id, NonPersistentDocumentFlags nonPersistentDocumentFlags, string changeVector)
{
if (nonPersistentDocumentFlags != NonPersistentDocumentFlags.None) // replication or engine running an operation, we can skip checking it
{
return;
}
if (_parent._documentDatabase.ClusterTransactionId == null)
{
return;
}
long indexFromChangeVector = ChangeVectorUtils.GetEtagById(changeVector, _parent._documentDatabase.ClusterTransactionId);
if (indexFromChangeVector == 0)
{
return;
}
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext clusterContext))
using (clusterContext.OpenReadTransaction())
{
var guardId = CompareExchangeKey.GetStorageKey(_parent._documentDatabase.Name, ClusterTransactionCommand.GetAtomicGuardKey(id));
var(indexFromCluster, val) = _serverStore.Cluster.GetCompareExchangeValue(clusterContext, guardId);
if (indexFromChangeVector != indexFromCluster)
{
throw new ConcurrencyException(
$"Cannot PUT document '{id}' because its change vector's cluster transaction index is set to {indexFromChangeVector} " +
$"but the compare exchange guard ('{ClusterTransactionCommand.GetAtomicGuardKey(id)}') is set to {indexFromCluster}")
{
Id = id
};
}
}
}
private long GetMinLastEtag()
{
var min = long.MaxValue;
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var record = _serverStore.Cluster.ReadRawDatabaseRecord(context, _database.Name);
foreach (var taskId in record.PeriodicBackupsTaskIds)
{
var config = record.GetPeriodicBackupConfiguration(taskId);
if (config.IncrementalBackupFrequency == null)
{
// if there is no status for this, we don't need to take into account tombstones
continue; // if the backup is always full, we don't need to take into account the tombstones, since we never back them up.
}
var status = GetBackupStatusFromCluster(_serverStore, context, _database.Name, taskId);
if (status == null)
{
// if there is no status for this, we don't need to take into account tombstones
return(0); // cannot delete the tombstones until we've done a full backup
}
var etag = ChangeVectorUtils.GetEtagById(status.LastDatabaseChangeVector, _database.DbBase64Id);
min = Math.Min(etag, min);
}
return(min);
}
}
public override int Execute(DocumentsOperationContext context)
{
if (Database.ServerStore.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable)
{
FeaturesAvailabilityException.Throw("Cluster Transactions");
}
var global = context.LastDatabaseChangeVector ??
(context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context));
var dbGrpId = Database.DatabaseGroupId;
var current = ChangeVectorUtils.GetEtagById(global, dbGrpId);
foreach (var command in _batch)
{
Replies.Add(command.Index, new DynamicJsonArray());
Reply = Replies[command.Index];
var commands = command.Commands;
var count = command.PreviousCount;
var options = Options[command.Index] = command.Options;
if (options.WaitForIndexesTimeout != null)
{
ModifiedCollections = new HashSet <string>();
}
if (commands != null)
{
foreach (BlittableJsonReaderObject blittableCommand in commands)
{
count++;
var changeVector = $"RAFT:{count}-{dbGrpId}";
var cmd = JsonDeserializationServer.ClusterTransactionDataCommand(blittableCommand);
switch (cmd.Type)
{
case CommandType.PUT:
if (current < count)
{
// delete the document to avoid exception if we put new document in a different collection.
// TODO: document this behavior
using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId))
{
Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, expectedChangeVector: null,
nonPersistentFlags: NonPersistentDocumentFlags.SkipRevisionCreation);
}
var putResult = Database.DocumentsStorage.Put(context, cmd.Id, null, cmd.Document.Clone(context), changeVector: changeVector,
flags: DocumentFlags.FromClusterTransaction);
context.DocumentDatabase.HugeDocuments.AddIfDocIsHuge(cmd.Id, cmd.Document.Size);
AddPutResult(putResult);
}
else
{
try
{
var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id);
if (item.Missing)
{
AddPutResult(new DocumentsStorage.PutOperationResults
{
ChangeVector = changeVector,
Id = cmd.Id,
LastModified = DateTime.UtcNow,
Collection = Database.DocumentsStorage.ExtractCollectionName(context, cmd.Document)
});
continue;
}
var collection = GetCollection(context, item);
AddPutResult(new DocumentsStorage.PutOperationResults
{
ChangeVector = changeVector,
Id = cmd.Id,
Flags = item.Document?.Flags ?? item.Tombstone.Flags,
LastModified = item.Document?.LastModified ?? item.Tombstone.LastModified,
Collection = collection
});
}
catch (DocumentConflictException)
{
AddPutResult(new DocumentsStorage.PutOperationResults
{
ChangeVector = changeVector,
Id = cmd.Id,
Collection = GetFirstConflictCollection(context, cmd)
});
}
}
break;
case CommandType.DELETE:
if (current < count)
{
using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId))
{
var deleteResult = Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, null, changeVector: changeVector,
documentFlags: DocumentFlags.FromClusterTransaction);
AddDeleteResult(deleteResult, cmd.Id);
}
}
else
{
try
{
var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id);
if (item.Missing)
{
AddDeleteResult(new DocumentsStorage.DeleteOperationResult
{
ChangeVector = changeVector,
Collection = null
}, cmd.Id);
continue;
}
var collection = GetCollection(context, item);
AddDeleteResult(new DocumentsStorage.DeleteOperationResult
{
ChangeVector = changeVector,
Collection = collection
}, cmd.Id);
}
catch (DocumentConflictException)
{
AddDeleteResult(new DocumentsStorage.DeleteOperationResult
{
ChangeVector = changeVector,
Collection = GetFirstConflictCollection(context, cmd)
}, cmd.Id);
}
}
break;
default:
throw new NotSupportedException($"{cmd.Type} is not supported in {nameof(ClusterTransactionMergedCommand)}.");
}
}
}
if (context.LastDatabaseChangeVector == null)
{
context.LastDatabaseChangeVector = global;
}
var result = ChangeVectorUtils.TryUpdateChangeVector("RAFT", dbGrpId, count, context.LastDatabaseChangeVector);
if (result.IsValid)
{
context.LastDatabaseChangeVector = result.ChangeVector;
}
}
return(Reply.Count);
}
public static async Task WaitForIndexesAsync(DocumentsContextPool contextPool, DocumentDatabase database, TimeSpan timeout,
List <string> specifiedIndexesQueryString, bool throwOnTimeout,
string lastChangeVector, long lastTombstoneEtag, HashSet <string> modifiedCollections)
{
// waitForIndexesTimeout=timespan & waitForIndexThrow=false (default true)
// waitForSpecificIndex=specific index1 & waitForSpecificIndex=specific index 2
if (modifiedCollections.Count == 0)
{
return;
}
var indexesToWait = new List <WaitForIndexItem>();
var indexesToCheck = GetImpactedIndexesToWaitForToBecomeNonStale(database, specifiedIndexesQueryString, modifiedCollections);
if (indexesToCheck.Count == 0)
{
return;
}
var sp = Stopwatch.StartNew();
// we take the awaiter _before_ the indexing transaction happens,
// so if there are any changes, it will already happen to it, and we'll
// query the index again. This is important because of:
// http://issues.hibernatingrhinos.com/issue/RavenDB-5576
foreach (var index in indexesToCheck)
{
var indexToWait = new WaitForIndexItem
{
Index = index,
IndexBatchAwaiter = index.GetIndexingBatchAwaiter(),
WaitForIndexing = new AsyncWaitForIndexing(sp, timeout, index)
};
indexesToWait.Add(indexToWait);
}
using (contextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
while (true)
{
var hadStaleIndexes = false;
using (context.OpenReadTransaction())
{
foreach (var waitForIndexItem in indexesToWait)
{
var lastEtag = lastChangeVector != null?ChangeVectorUtils.GetEtagById(lastChangeVector, database.DbBase64Id) : 0;
var cutoffEtag = Math.Max(lastEtag, lastTombstoneEtag);
if (waitForIndexItem.Index.IsStale(context, cutoffEtag) == false)
{
continue;
}
hadStaleIndexes = true;
await waitForIndexItem.WaitForIndexing.WaitForIndexingAsync(waitForIndexItem.IndexBatchAwaiter);
if (waitForIndexItem.WaitForIndexing.TimeoutExceeded && throwOnTimeout)
{
throw new TimeoutException(
$"After waiting for {sp.Elapsed}, could not verify that {indexesToCheck.Count} " +
$"indexes has caught up with the changes as of etag: {cutoffEtag}");
}
}
}
if (hadStaleIndexes == false)
{
return;
}
}
}
}
private bool EnsureReplicating(IDocumentStore src, IDocumentStore dst, TimeSpan timeout, out string errorMsg)
{
errorMsg = string.Empty;
var id = "marker/" + Guid.NewGuid();
string cv;
using (var s = src.OpenSession())
{
var entity = new { };
s.Store(entity, id);
s.SaveChanges();
cv = s.Advanced.GetChangeVectorFor(entity);
}
if (WaitForDocumentToReplicate <object>(dst, id, (int)timeout.TotalMilliseconds) != null)
{
return(true);
}
var sb = new StringBuilder()
.AppendLine($"Failed to replicate from '{src.Database}' to '{dst.Database}'");
var database = Databases.GetDocumentDatabaseInstanceFor(src).Result;
var etag = ChangeVectorUtils.GetEtagById(cv, database.DbBase64Id);
Console.WriteLine($"document etag : {etag}");
var outgoingFailureInfo = database.ReplicationLoader.OutgoingFailureInfo.ToList();
if (outgoingFailureInfo.Count > 0)
{
sb.AppendLine($"{src.Database} Outgoing Failure Info : ");
foreach (var e in outgoingFailureInfo)
{
sb.AppendLine(string.Join(", ", e.Value.Errors.Select(x => x.Message)));
}
sb.AppendLine();
}
foreach (var store in new[] { src, dst })
{
var stats = store.Maintenance.Send(new GetReplicationPerformanceStatisticsOperation());
var incoming = stats.Incoming.FirstOrDefault();
if (incoming != null)
{
sb.AppendLine($"{store.Database} Incoming Performance Stats : ");
foreach (var s in incoming.Performance)
{
sb.AppendLine(ToString(s));
}
sb.AppendLine();
}
var outgoing = stats.Outgoing.FirstOrDefault();
if (outgoing != null)
{
sb.AppendLine($"{store.Database} Outgoing Performance Stats : ");
foreach (var s in outgoing.Performance)
{
sb.AppendLine(ToString(s));
}
sb.AppendLine();
}
}
errorMsg = sb.ToString();
return(false);
}
public bool ExecuteReplicationOnce(TcpConnectionOptions tcpConnectionOptions, OutgoingReplicationStatsScope stats, ref long next)
{
EnsureValidStats(stats);
var wasInterrupted = false;
var delay = GetDelayReplication();
var currentNext = next;
using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
using (documentsContext.OpenReadTransaction())
{
try
{
// we scan through the documents to send to the other side, we need to be careful about
// filtering a lot of documents, because we need to let the other side know about this, and
// at the same time, we need to send a heartbeat to keep the tcp connection alive
_lastEtag = _parent._lastSentDocumentEtag;
var lastEtagFromDestinationChangeVector = ChangeVectorUtils.GetEtagById(_parent.LastAcceptedChangeVector, _parent._database.DbBase64Id);
if (lastEtagFromDestinationChangeVector > _lastEtag)
{
if (_log.IsInfoEnabled)
{
_log.Info($"We jump to get items from etag {lastEtagFromDestinationChangeVector} instead of {_lastEtag}, because we got a bigger etag for the destination database change vector ({_parent.LastAcceptedChangeVector})");
}
_lastEtag = lastEtagFromDestinationChangeVector;
}
_parent.CancellationToken.ThrowIfCancellationRequested();
var skippedReplicationItemsInfo = new SkippedReplicationItemsInfo();
long prevLastEtag = _lastEtag;
var replicationState = new ReplicationState
{
BatchSize = _parent._database.Configuration.Replication.MaxItemsCount,
MaxSizeToSend = _parent._database.Configuration.Replication.MaxSizeToSend,
CurrentNext = currentNext,
Delay = delay,
Context = documentsContext,
LastTransactionMarker = -1,
NumberOfItemsSent = 0,
Size = 0L
};
using (_stats.Storage.Start())
{
foreach (var item in GetReplicationItems(_parent._database, documentsContext, _lastEtag, _stats, _parent.SupportedFeatures.Replication.CaseInsensitiveCounters))
{
_parent.ForTestingPurposes?.OnDocumentSenderFetchNewItem?.Invoke();
_parent.CancellationToken.ThrowIfCancellationRequested();
AssertNoLegacyReplicationViolation(item);
if (replicationState.LastTransactionMarker != item.TransactionMarker)
{
replicationState.Item = item;
if (CanContinueBatch(replicationState, ref next) == false)
{
wasInterrupted = true;
break;
}
replicationState.LastTransactionMarker = item.TransactionMarker;
}
_stats.Storage.RecordInputAttempt();
// here we add missing attachments in the same batch as the document that contains them without modifying the last etag or transaction boundary
if (MissingAttachmentsInLastBatch &&
item.Type == ReplicationBatchItem.ReplicationItemType.Document &&
item is DocumentReplicationItem docItem &&
docItem.Flags.Contain(DocumentFlags.HasAttachments))
{
var missingAttachmentBase64Hashes = replicationState.MissingAttachmentBase64Hashes ??= new HashSet <Slice>(SliceStructComparer.Instance);
var type = (docItem.Flags & DocumentFlags.Revision) == DocumentFlags.Revision ? AttachmentType.Revision : AttachmentType.Document;
foreach (var attachment in _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentsForDocument(documentsContext, type, docItem.Id, docItem.ChangeVector))
{
// we need to filter attachments that are been sent in the same batch as the document
if (attachment.Etag >= prevLastEtag)
{
if (_replicaAttachmentStreams.ContainsKey(attachment.Base64Hash) == false)
{
missingAttachmentBase64Hashes.Add(attachment.Base64Hash);
}
continue;
}
var stream = _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentStream(documentsContext, attachment.Base64Hash);
attachment.Stream = stream;
var attachmentItem = AttachmentReplicationItem.From(documentsContext, attachment);
AddReplicationItemToBatch(attachmentItem, _stats.Storage, replicationState, skippedReplicationItemsInfo);
replicationState.Size += attachmentItem.Size;
}
}
_lastEtag = item.Etag;
if (AddReplicationItemToBatch(item, _stats.Storage, replicationState, skippedReplicationItemsInfo) == false)
{
// this item won't be needed anymore
item.Dispose();
continue;
}
replicationState.Size += item.Size;
replicationState.NumberOfItemsSent++;
}
}
if (_log.IsInfoEnabled)
{
if (skippedReplicationItemsInfo.SkippedItems > 0)
{
var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector);
_log.Info(message);
}
var msg = $"Found {_orderedReplicaItems.Count:#,#;;0} documents " +
$"and {_replicaAttachmentStreams.Count} attachment's streams " +
$"to replicate to {_parent.Node.FromString()}, ";
var encryptionSize = documentsContext.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes);
if (encryptionSize > 0)
{
msg += $"encryption buffer overhead size is {new Size(encryptionSize, SizeUnit.Bytes)}, ";
}
msg += $"total size: {new Size(replicationState.Size + encryptionSize, SizeUnit.Bytes)}";
_log.Info(msg);
}
if (_orderedReplicaItems.Count == 0)
{
var hasModification = _lastEtag != _parent._lastSentDocumentEtag;
// ensure that the other server is aware that we skipped
// on (potentially a lot of) documents to send, and we update
// the last etag they have from us on the other side
_parent._lastSentDocumentEtag = _lastEtag;
_parent._lastDocumentSentTime = DateTime.UtcNow;
var changeVector = wasInterrupted ? null : DocumentsStorage.GetDatabaseChangeVector(documentsContext);
_parent.SendHeartbeat(changeVector);
return(hasModification);
}
_parent.CancellationToken.ThrowIfCancellationRequested();
MissingAttachmentsInLastBatch = false;
try
{
using (_stats.Network.Start())
{
SendDocumentsBatch(documentsContext, _stats.Network);
tcpConnectionOptions._lastEtagSent = _lastEtag;
tcpConnectionOptions.RegisterBytesSent(replicationState.Size);
if (MissingAttachmentsInLastBatch)
{
return(false);
}
}
}
catch (OperationCanceledException)
{
if (_log.IsInfoEnabled)
{
_log.Info("Received cancellation notification while sending document replication batch.");
}
throw;
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info("Failed to send document replication batch", e);
}
throw;
}
return(true);
}
finally
{
foreach (var item in _orderedReplicaItems)
{
item.Value.Dispose();
}
_orderedReplicaItems.Clear();
_replicaAttachmentStreams.Clear();
}
}
}