public void ValidateAtomicGuard(string id, NonPersistentDocumentFlags nonPersistentDocumentFlags, string changeVector) { if (nonPersistentDocumentFlags != NonPersistentDocumentFlags.None) // replication or engine running an operation, we can skip checking it { return; } if (_parent._documentDatabase.ClusterTransactionId == null) { return; } long indexFromChangeVector = ChangeVectorUtils.GetEtagById(changeVector, _parent._documentDatabase.ClusterTransactionId); if (indexFromChangeVector == 0) { return; } using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext clusterContext)) using (clusterContext.OpenReadTransaction()) { var guardId = CompareExchangeKey.GetStorageKey(_parent._documentDatabase.Name, ClusterTransactionCommand.GetAtomicGuardKey(id)); var(indexFromCluster, val) = _serverStore.Cluster.GetCompareExchangeValue(clusterContext, guardId); if (indexFromChangeVector != indexFromCluster) { throw new ConcurrencyException( $"Cannot PUT document '{id}' because its change vector's cluster transaction index is set to {indexFromChangeVector} " + $"but the compare exchange guard ('{ClusterTransactionCommand.GetAtomicGuardKey(id)}') is set to {indexFromCluster}") { Id = id }; } } }
private long GetMinLastEtag() { var min = long.MaxValue; using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) { var record = _serverStore.Cluster.ReadRawDatabaseRecord(context, _database.Name); foreach (var taskId in record.PeriodicBackupsTaskIds) { var config = record.GetPeriodicBackupConfiguration(taskId); if (config.IncrementalBackupFrequency == null) { // if there is no status for this, we don't need to take into account tombstones continue; // if the backup is always full, we don't need to take into account the tombstones, since we never back them up. } var status = GetBackupStatusFromCluster(_serverStore, context, _database.Name, taskId); if (status == null) { // if there is no status for this, we don't need to take into account tombstones return(0); // cannot delete the tombstones until we've done a full backup } var etag = ChangeVectorUtils.GetEtagById(status.LastDatabaseChangeVector, _database.DbBase64Id); min = Math.Min(etag, min); } return(min); } }
public override int Execute(DocumentsOperationContext context) { if (Database.ServerStore.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable) { FeaturesAvailabilityException.Throw("Cluster Transactions"); } var global = context.LastDatabaseChangeVector ?? (context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context)); var dbGrpId = Database.DatabaseGroupId; var current = ChangeVectorUtils.GetEtagById(global, dbGrpId); foreach (var command in _batch) { Replies.Add(command.Index, new DynamicJsonArray()); Reply = Replies[command.Index]; var commands = command.Commands; var count = command.PreviousCount; var options = Options[command.Index] = command.Options; if (options.WaitForIndexesTimeout != null) { ModifiedCollections = new HashSet <string>(); } if (commands != null) { foreach (BlittableJsonReaderObject blittableCommand in commands) { count++; var changeVector = $"RAFT:{count}-{dbGrpId}"; var cmd = JsonDeserializationServer.ClusterTransactionDataCommand(blittableCommand); switch (cmd.Type) { case CommandType.PUT: if (current < count) { // delete the document to avoid exception if we put new document in a different collection. // TODO: document this behavior using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId)) { Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, expectedChangeVector: null, nonPersistentFlags: NonPersistentDocumentFlags.SkipRevisionCreation); } var putResult = Database.DocumentsStorage.Put(context, cmd.Id, null, cmd.Document.Clone(context), changeVector: changeVector, flags: DocumentFlags.FromClusterTransaction); context.DocumentDatabase.HugeDocuments.AddIfDocIsHuge(cmd.Id, cmd.Document.Size); AddPutResult(putResult); } else { try { var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id); if (item.Missing) { AddPutResult(new DocumentsStorage.PutOperationResults { ChangeVector = changeVector, Id = cmd.Id, LastModified = DateTime.UtcNow, Collection = Database.DocumentsStorage.ExtractCollectionName(context, cmd.Document) }); continue; } var collection = GetCollection(context, item); AddPutResult(new DocumentsStorage.PutOperationResults { ChangeVector = changeVector, Id = cmd.Id, Flags = item.Document?.Flags ?? item.Tombstone.Flags, LastModified = item.Document?.LastModified ?? item.Tombstone.LastModified, Collection = collection }); } catch (DocumentConflictException) { AddPutResult(new DocumentsStorage.PutOperationResults { ChangeVector = changeVector, Id = cmd.Id, Collection = GetFirstConflictCollection(context, cmd) }); } } break; case CommandType.DELETE: if (current < count) { using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId)) { var deleteResult = Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, null, changeVector: changeVector, documentFlags: DocumentFlags.FromClusterTransaction); AddDeleteResult(deleteResult, cmd.Id); } } else { try { var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id); if (item.Missing) { AddDeleteResult(new DocumentsStorage.DeleteOperationResult { ChangeVector = changeVector, Collection = null }, cmd.Id); continue; } var collection = GetCollection(context, item); AddDeleteResult(new DocumentsStorage.DeleteOperationResult { ChangeVector = changeVector, Collection = collection }, cmd.Id); } catch (DocumentConflictException) { AddDeleteResult(new DocumentsStorage.DeleteOperationResult { ChangeVector = changeVector, Collection = GetFirstConflictCollection(context, cmd) }, cmd.Id); } } break; default: throw new NotSupportedException($"{cmd.Type} is not supported in {nameof(ClusterTransactionMergedCommand)}."); } } } if (context.LastDatabaseChangeVector == null) { context.LastDatabaseChangeVector = global; } var result = ChangeVectorUtils.TryUpdateChangeVector("RAFT", dbGrpId, count, context.LastDatabaseChangeVector); if (result.IsValid) { context.LastDatabaseChangeVector = result.ChangeVector; } } return(Reply.Count); }
public static async Task WaitForIndexesAsync(DocumentsContextPool contextPool, DocumentDatabase database, TimeSpan timeout, List <string> specifiedIndexesQueryString, bool throwOnTimeout, string lastChangeVector, long lastTombstoneEtag, HashSet <string> modifiedCollections) { // waitForIndexesTimeout=timespan & waitForIndexThrow=false (default true) // waitForSpecificIndex=specific index1 & waitForSpecificIndex=specific index 2 if (modifiedCollections.Count == 0) { return; } var indexesToWait = new List <WaitForIndexItem>(); var indexesToCheck = GetImpactedIndexesToWaitForToBecomeNonStale(database, specifiedIndexesQueryString, modifiedCollections); if (indexesToCheck.Count == 0) { return; } var sp = Stopwatch.StartNew(); // we take the awaiter _before_ the indexing transaction happens, // so if there are any changes, it will already happen to it, and we'll // query the index again. This is important because of: // http://issues.hibernatingrhinos.com/issue/RavenDB-5576 foreach (var index in indexesToCheck) { var indexToWait = new WaitForIndexItem { Index = index, IndexBatchAwaiter = index.GetIndexingBatchAwaiter(), WaitForIndexing = new AsyncWaitForIndexing(sp, timeout, index) }; indexesToWait.Add(indexToWait); } using (contextPool.AllocateOperationContext(out DocumentsOperationContext context)) { while (true) { var hadStaleIndexes = false; using (context.OpenReadTransaction()) { foreach (var waitForIndexItem in indexesToWait) { var lastEtag = lastChangeVector != null?ChangeVectorUtils.GetEtagById(lastChangeVector, database.DbBase64Id) : 0; var cutoffEtag = Math.Max(lastEtag, lastTombstoneEtag); if (waitForIndexItem.Index.IsStale(context, cutoffEtag) == false) { continue; } hadStaleIndexes = true; await waitForIndexItem.WaitForIndexing.WaitForIndexingAsync(waitForIndexItem.IndexBatchAwaiter); if (waitForIndexItem.WaitForIndexing.TimeoutExceeded && throwOnTimeout) { throw new TimeoutException( $"After waiting for {sp.Elapsed}, could not verify that {indexesToCheck.Count} " + $"indexes has caught up with the changes as of etag: {cutoffEtag}"); } } } if (hadStaleIndexes == false) { return; } } } }
private bool EnsureReplicating(IDocumentStore src, IDocumentStore dst, TimeSpan timeout, out string errorMsg) { errorMsg = string.Empty; var id = "marker/" + Guid.NewGuid(); string cv; using (var s = src.OpenSession()) { var entity = new { }; s.Store(entity, id); s.SaveChanges(); cv = s.Advanced.GetChangeVectorFor(entity); } if (WaitForDocumentToReplicate <object>(dst, id, (int)timeout.TotalMilliseconds) != null) { return(true); } var sb = new StringBuilder() .AppendLine($"Failed to replicate from '{src.Database}' to '{dst.Database}'"); var database = Databases.GetDocumentDatabaseInstanceFor(src).Result; var etag = ChangeVectorUtils.GetEtagById(cv, database.DbBase64Id); Console.WriteLine($"document etag : {etag}"); var outgoingFailureInfo = database.ReplicationLoader.OutgoingFailureInfo.ToList(); if (outgoingFailureInfo.Count > 0) { sb.AppendLine($"{src.Database} Outgoing Failure Info : "); foreach (var e in outgoingFailureInfo) { sb.AppendLine(string.Join(", ", e.Value.Errors.Select(x => x.Message))); } sb.AppendLine(); } foreach (var store in new[] { src, dst }) { var stats = store.Maintenance.Send(new GetReplicationPerformanceStatisticsOperation()); var incoming = stats.Incoming.FirstOrDefault(); if (incoming != null) { sb.AppendLine($"{store.Database} Incoming Performance Stats : "); foreach (var s in incoming.Performance) { sb.AppendLine(ToString(s)); } sb.AppendLine(); } var outgoing = stats.Outgoing.FirstOrDefault(); if (outgoing != null) { sb.AppendLine($"{store.Database} Outgoing Performance Stats : "); foreach (var s in outgoing.Performance) { sb.AppendLine(ToString(s)); } sb.AppendLine(); } } errorMsg = sb.ToString(); return(false); }
public bool ExecuteReplicationOnce(TcpConnectionOptions tcpConnectionOptions, OutgoingReplicationStatsScope stats, ref long next) { EnsureValidStats(stats); var wasInterrupted = false; var delay = GetDelayReplication(); var currentNext = next; using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext)) using (documentsContext.OpenReadTransaction()) { try { // we scan through the documents to send to the other side, we need to be careful about // filtering a lot of documents, because we need to let the other side know about this, and // at the same time, we need to send a heartbeat to keep the tcp connection alive _lastEtag = _parent._lastSentDocumentEtag; var lastEtagFromDestinationChangeVector = ChangeVectorUtils.GetEtagById(_parent.LastAcceptedChangeVector, _parent._database.DbBase64Id); if (lastEtagFromDestinationChangeVector > _lastEtag) { if (_log.IsInfoEnabled) { _log.Info($"We jump to get items from etag {lastEtagFromDestinationChangeVector} instead of {_lastEtag}, because we got a bigger etag for the destination database change vector ({_parent.LastAcceptedChangeVector})"); } _lastEtag = lastEtagFromDestinationChangeVector; } _parent.CancellationToken.ThrowIfCancellationRequested(); var skippedReplicationItemsInfo = new SkippedReplicationItemsInfo(); long prevLastEtag = _lastEtag; var replicationState = new ReplicationState { BatchSize = _parent._database.Configuration.Replication.MaxItemsCount, MaxSizeToSend = _parent._database.Configuration.Replication.MaxSizeToSend, CurrentNext = currentNext, Delay = delay, Context = documentsContext, LastTransactionMarker = -1, NumberOfItemsSent = 0, Size = 0L }; using (_stats.Storage.Start()) { foreach (var item in GetReplicationItems(_parent._database, documentsContext, _lastEtag, _stats, _parent.SupportedFeatures.Replication.CaseInsensitiveCounters)) { _parent.ForTestingPurposes?.OnDocumentSenderFetchNewItem?.Invoke(); _parent.CancellationToken.ThrowIfCancellationRequested(); AssertNoLegacyReplicationViolation(item); if (replicationState.LastTransactionMarker != item.TransactionMarker) { replicationState.Item = item; if (CanContinueBatch(replicationState, ref next) == false) { wasInterrupted = true; break; } replicationState.LastTransactionMarker = item.TransactionMarker; } _stats.Storage.RecordInputAttempt(); // here we add missing attachments in the same batch as the document that contains them without modifying the last etag or transaction boundary if (MissingAttachmentsInLastBatch && item.Type == ReplicationBatchItem.ReplicationItemType.Document && item is DocumentReplicationItem docItem && docItem.Flags.Contain(DocumentFlags.HasAttachments)) { var missingAttachmentBase64Hashes = replicationState.MissingAttachmentBase64Hashes ??= new HashSet <Slice>(SliceStructComparer.Instance); var type = (docItem.Flags & DocumentFlags.Revision) == DocumentFlags.Revision ? AttachmentType.Revision : AttachmentType.Document; foreach (var attachment in _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentsForDocument(documentsContext, type, docItem.Id, docItem.ChangeVector)) { // we need to filter attachments that are been sent in the same batch as the document if (attachment.Etag >= prevLastEtag) { if (_replicaAttachmentStreams.ContainsKey(attachment.Base64Hash) == false) { missingAttachmentBase64Hashes.Add(attachment.Base64Hash); } continue; } var stream = _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentStream(documentsContext, attachment.Base64Hash); attachment.Stream = stream; var attachmentItem = AttachmentReplicationItem.From(documentsContext, attachment); AddReplicationItemToBatch(attachmentItem, _stats.Storage, replicationState, skippedReplicationItemsInfo); replicationState.Size += attachmentItem.Size; } } _lastEtag = item.Etag; if (AddReplicationItemToBatch(item, _stats.Storage, replicationState, skippedReplicationItemsInfo) == false) { // this item won't be needed anymore item.Dispose(); continue; } replicationState.Size += item.Size; replicationState.NumberOfItemsSent++; } } if (_log.IsInfoEnabled) { if (skippedReplicationItemsInfo.SkippedItems > 0) { var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector); _log.Info(message); } var msg = $"Found {_orderedReplicaItems.Count:#,#;;0} documents " + $"and {_replicaAttachmentStreams.Count} attachment's streams " + $"to replicate to {_parent.Node.FromString()}, "; var encryptionSize = documentsContext.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes); if (encryptionSize > 0) { msg += $"encryption buffer overhead size is {new Size(encryptionSize, SizeUnit.Bytes)}, "; } msg += $"total size: {new Size(replicationState.Size + encryptionSize, SizeUnit.Bytes)}"; _log.Info(msg); } if (_orderedReplicaItems.Count == 0) { var hasModification = _lastEtag != _parent._lastSentDocumentEtag; // ensure that the other server is aware that we skipped // on (potentially a lot of) documents to send, and we update // the last etag they have from us on the other side _parent._lastSentDocumentEtag = _lastEtag; _parent._lastDocumentSentTime = DateTime.UtcNow; var changeVector = wasInterrupted ? null : DocumentsStorage.GetDatabaseChangeVector(documentsContext); _parent.SendHeartbeat(changeVector); return(hasModification); } _parent.CancellationToken.ThrowIfCancellationRequested(); MissingAttachmentsInLastBatch = false; try { using (_stats.Network.Start()) { SendDocumentsBatch(documentsContext, _stats.Network); tcpConnectionOptions._lastEtagSent = _lastEtag; tcpConnectionOptions.RegisterBytesSent(replicationState.Size); if (MissingAttachmentsInLastBatch) { return(false); } } } catch (OperationCanceledException) { if (_log.IsInfoEnabled) { _log.Info("Received cancellation notification while sending document replication batch."); } throw; } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info("Failed to send document replication batch", e); } throw; } return(true); } finally { foreach (var item in _orderedReplicaItems) { item.Value.Dispose(); } _orderedReplicaItems.Clear(); _replicaAttachmentStreams.Clear(); } } }