Esempio n. 1
0
        private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder, out int replicatedDocuments)
        {
            replicatedDocuments = 0;
            JsonDocumentsToReplicate documentsToReplicate = null;
            var sp = Stopwatch.StartNew();
            IDisposable removeBatch = null;

            var prefetchingBehavior = prefetchingBehaviors.GetOrAdd(destination.ConnectionStringOptions.Url,
                x => docDb.Prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Replicator, autoTuner, string.Format("Replication for URL: {0}" ,destination.ConnectionStringOptions.DefaultDatabase) ));


            prefetchingBehavior.AdditionalInfo = string.Format("For destination: {0}. Last replicated etag: {1}", destination.ConnectionStringOptions.Url, destinationsReplicationInformationForSource.LastDocumentEtag);

            try
            {
                using (var scope = recorder.StartRecording("Get"))
                {
                    documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, prefetchingBehavior, scope);
                    if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
                    {
                        if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
                        {
                            // we don't notify remote server about updates to system docs, see: RavenDB-715
                            if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0
                                || documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate
                                || documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555
                            {
                                using (scope.StartRecording("Notify"))
                                {
                                    SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
                                    scope.Record(new RavenJObject
                                             {
                                                 { "LastDocEtag", documentsToReplicate.LastEtag.ToString() }
                                             });
                                }
                            }
                        }
                        RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag);
                        return null;
                    }
                }

                // if the db is idling in all respect except sending out replication, let us keep it that way.
                docDb.WorkContext.UpdateFoundWork();

                removeBatch = prefetchingBehavior.UpdateCurrentlyUsedBatches(documentsToReplicate.LoadedDocs);

                using (var scope = recorder.StartRecording("Send"))
                {
                    string lastError;
                    if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy
                    {
                        if (IsFirstFailure(destination.ConnectionStringOptions.Url))
                        {
                            log.Info(
                                "This is the first failure for {0}, assuming transient failure and trying again",
                                destination);
                            if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail
                            {
                                RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
                                return true;
                            }
                        }
                        // if we had an error sending to this endpoint, it might be because we are sending too much data, or because
                        // the request timed out. This will let us know that the next time we try, we'll use just the initial doc counts
                        // and we'll be much more conservative with increasing the sizes
                        prefetchingBehavior.OutOfMemoryExceptionHappened();
                        scope.RecordError(lastError);
                        RecordFailure(destination.ConnectionStringOptions.Url, lastError);
                        return false;
                    }
                }
            }
            finally
            {
                if (documentsToReplicate != null && documentsToReplicate.LoadedDocs != null)
                {
                    prefetchingBehavior.UpdateAutoThrottler(documentsToReplicate.LoadedDocs, sp.Elapsed);
                    replicatedDocuments = documentsToReplicate.LoadedDocs.Count;
                }

                if (removeBatch != null)
                    removeBatch.Dispose();
            }

            RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
            return true;
        }
Esempio n. 2
0
        private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder)
        {
            Tuple<RavenJArray, Etag> tuple;
            RavenJArray attachments;

            using (var scope = recorder.StartRecording("Get"))
            {
                tuple = GetAttachments(destinationsReplicationInformationForSource, destination, scope);
                attachments = tuple.Item1;

                if (attachments == null || attachments.Length == 0)
                {
                    if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag)
                    {
                        SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2);
                    }
                    return null;
                }
            }

            using (var scope = recorder.StartRecording("Send"))
            {
                string lastError;
                if (TryReplicationAttachments(destination, attachments, out lastError) == false) // failed to replicate, start error handling strategy
                {
                    if (IsFirstFailure(destination.ConnectionStringOptions.Url))
                    {
                        log.Info("This is the first failure for {0}, assuming transient failure and trying again", destination);
                        if (TryReplicationAttachments(destination, attachments, out lastError)) // success on second fail
                        {
                            RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2, forDocuments:false);
                            return true;
                        }
                    }

                    scope.RecordError(lastError);
                    RecordFailure(destination.ConnectionStringOptions.Url, lastError);
                    return false;
                }
            }

            RecordSuccess(destination.ConnectionStringOptions.Url,
                lastReplicatedEtag: tuple.Item2, forDocuments: false);

            return true;
        }
Esempio n. 3
0
        private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, PrefetchingBehavior prefetchingBehavior, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
        {
            var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds);
            var duration = Stopwatch.StartNew();
            var result = new JsonDocumentsToReplicate
            {
                LastEtag = Etag.Empty,
            };
            try
            {
                var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
                var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch;

                docDb.TransactionalStorage.Batch(actions =>
                {
                    var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;

                    int docsSinceLastReplEtag = 0;
                    List<JsonDocument> docsToReplicate;
                    List<JsonDocument> filteredDocsToReplicate;
                    result.LastEtag = lastEtag;

                    while (true)
                    {
                        docDb.WorkContext.CancellationToken.ThrowIfCancellationRequested();

                        docsToReplicate = GetDocsToReplicate(actions, prefetchingBehavior, result, maxNumberOfItemsToReceiveInSingleBatch);

                        filteredDocsToReplicate =
                            docsToReplicate
                                .Where(document =>
                                {
                                    var info = docDb.Documents.GetRecentTouchesFor(document.Key);
                                    if (info != null)
                                    {
                                        if (info.TouchedEtag.CompareTo(result.LastEtag) > 0)
                                        {
                                            log.Debug(
                                                "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
                                                document.Key, destinationId, info.TouchedEtag);
                                            return false;
                                        }
                                    }

                                    string reason;
                                    return destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) &&
                                           prefetchingBehavior.FilterDocuments(document);
                                })
                                .ToList();

                        docsSinceLastReplEtag += docsToReplicate.Count;
                        result.CountOfFilteredDocumentsWhichAreSystemDocuments +=
                            docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
                        result.CountOfFilteredDocumentsWhichOriginFromDestination +=
                            docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata));

                        if (docsToReplicate.Count > 0)
                        {
                            var lastDoc = docsToReplicate.Last();
                            Debug.Assert(lastDoc.Etag != null);
                            result.LastEtag = lastDoc.Etag;
                            if (lastDoc.LastModified.HasValue)
                                result.LastLastModified = lastDoc.LastModified.Value;
                        }

                        if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
                        {
                            break;
                        }

                        log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);

                        if (duration.Elapsed > timeout)
                            break;
                    }

                    log.Debug(() =>
                    {
                        if (docsSinceLastReplEtag == 0)
                            return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
                                lastEtag);

                        if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
                            return string.Format("Replicating {0} docs [>{1}] to {2}.",
                                docsSinceLastReplEtag,
                                lastEtag,
                                destination);

                        var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
                        return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
                            docsSinceLastReplEtag,
                            filteredDocsToReplicate.Count,
                            destination,
                            string.Join(", ", diff),
                            lastEtag);
                    });

                    scope.Record(new RavenJObject
                    {
                        {"StartEtag", lastEtag.ToString()},
                        {"EndEtag", result.LastEtag.ToString()},
                        {"Count", docsSinceLastReplEtag},
                        {"FilteredCount", filteredDocsToReplicate.Count}
                    });

                    result.LoadedDocs = filteredDocsToReplicate;
                    docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeMetric(destination).Mark(docsSinceLastReplEtag);
                    docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeHistogram(destination).Update(docsSinceLastReplEtag);

                    result.Documents = new RavenJArray(filteredDocsToReplicate
                        .Select(x =>
                        {
                            JsonDocument.EnsureIdInMetadata(x);
                            EnsureReplicationInformationInMetadata(x.Metadata, docDb);
                            return x;
                        })
                        .Select(x => x.ToJson()));
                });
            }
            catch (Exception e)
            {
                scope.RecordError(e);
                log.WarnException(
                    "Could not get documents to replicate after: " +
                    destinationsReplicationInformationForSource.LastDocumentEtag, e);
            }
            return result;
        }
Esempio n. 4
0
        private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
        {
            var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds);
            var duration = Stopwatch.StartNew();

            RavenJArray attachments = null;
            Etag lastAttachmentEtag = Etag.Empty;
            try
            {
                var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
                var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch;

                docDb.TransactionalStorage.Batch(actions =>
                {
                    int attachmentSinceLastEtag = 0;
                    List<AttachmentInformation> attachmentsToReplicate;
                    List<AttachmentInformation> filteredAttachmentsToReplicate;
                    var startEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
                    lastAttachmentEtag = startEtag;
                    while (true)
                    {
                        attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag, maxNumberOfItemsToReceiveInSingleBatch);

                        filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();

                        attachmentSinceLastEtag += attachmentsToReplicate.Count;

                        if (attachmentsToReplicate.Count == 0 ||
                            filteredAttachmentsToReplicate.Count != 0)
                        {
                            break;
                        }

                        AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
                        Etag attachmentEtag = jsonDocument.Etag;
                        log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
                        lastAttachmentEtag = attachmentEtag;

                        if (duration.Elapsed > timeout)
                            break;
                    }

                    log.Debug(() =>
                    {
                        if (attachmentSinceLastEtag == 0)
                            return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
                                                 destinationsReplicationInformationForSource.LastAttachmentEtag);

                        if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
                            return string.Format("Replicating {0} attachments [>{1}] to {2}.",
                                             attachmentSinceLastEtag,
                                             destinationsReplicationInformationForSource.LastAttachmentEtag,
                                             destination);

                        var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
                        return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
                                             attachmentSinceLastEtag,
                                             filteredAttachmentsToReplicate.Count,
                                             destination,
                                             string.Join(", ", diff),
                                             destinationsReplicationInformationForSource.LastAttachmentEtag);
                    });

                    scope.Record(new RavenJObject
                                 {
                                     {"StartEtag", startEtag.ToString()},
                                     {"EndEtag", lastAttachmentEtag.ToString()},
                                     {"Count", attachmentSinceLastEtag},
                                     {"FilteredCount", filteredAttachmentsToReplicate.Count}
                                 });

                    attachments = new RavenJArray(filteredAttachmentsToReplicate
                                                      .Select(x =>
                                                      {
                                                          var data = new byte[0];
                                                          if (x.Size > 0)
                                                          {
                                                              data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
                                                          }

                                                          EnsureReplicationInformationInMetadata(x.Metadata, docDb);

                                                          return new RavenJObject
                                                                       {
                                                                           {"@metadata", x.Metadata},
                                                                           {"@id", x.Key},
                                                                           {"@etag", x.Etag.ToByteArray()},
                                                                           {"data", data}
                                                                       };
                                                      }));
                });
            }
            catch (InvalidDataException e)
            {
                RecordFailure(String.Empty, string.Format("Data is corrupted, could not proceed with attachment replication. Exception : {0}", e));
                scope.RecordError(e);
                log.ErrorException("Data is corrupted, could not proceed with replication", e);
            }
            catch (Exception e)
            {
                log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
            }
            return Tuple.Create(attachments, lastAttachmentEtag);
        }
Esempio n. 5
0
		public HttpResponseMessage ReplicationLastEtagGet()
		{
			string src;
			string dbid;
			var result = GetValuesForLastEtag(out src, out dbid);
			if (result != null)
				return result;

			using (Database.DisableAllTriggersForCurrentThread())
			{
				JsonDocument document = null;
				SourceReplicationInformationWithBatchInformation sourceReplicationInformation = null;

				var localServerInstanceId = Database.TransactionalStorage.Id; // this is my id, sent to the remote server

				if (string.IsNullOrEmpty(dbid))
				{
					// backward compatibility for replication behavior
					int nextStart = 0;
					var replicationSources = Database.Documents.GetDocumentsWithIdStartingWith(Constants.RavenReplicationSourcesBasePath, null, null, 0, int.MaxValue, CancellationToken.None, ref nextStart);
					foreach (RavenJObject replicationSource in replicationSources)
					{
						sourceReplicationInformation = replicationSource.JsonDeserialization<SourceReplicationInformationWithBatchInformation>();
						if (string.Equals(sourceReplicationInformation.Source, src, StringComparison.OrdinalIgnoreCase) == false)
							continue;

						document = replicationSource.ToJsonDocument();
						break;
					}
				}
				else
				{
					var remoteServerInstanceId = Guid.Parse(dbid);

					document = Database.Documents.Get(Constants.RavenReplicationSourcesBasePath + "/" + remoteServerInstanceId, null);
					if (document == null)
					{
						// migrate
						document = Database.Documents.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null);
						if (document != null)
						{
							sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformationWithBatchInformation>();
							Database.Documents.Put(Constants.RavenReplicationSourcesBasePath + "/" + sourceReplicationInformation.ServerInstanceId, Etag.Empty, document.DataAsJson, document.Metadata, null);
							Database.Documents.Delete(Constants.RavenReplicationSourcesBasePath + "/" + src, document.Etag, null);

							if (remoteServerInstanceId != sourceReplicationInformation.ServerInstanceId) 
								document = null;
						}
					}
				}

				if (document == null)
				{
					sourceReplicationInformation = new SourceReplicationInformationWithBatchInformation
					{
						Source = src,
						ServerInstanceId = localServerInstanceId,
						LastModified = SystemTime.UtcNow
					};
				}
				else
				{
					if (sourceReplicationInformation == null)
						sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformationWithBatchInformation>();

					if (string.Equals(sourceReplicationInformation.Source, src, StringComparison.OrdinalIgnoreCase) == false 
						&& sourceReplicationInformation.LastModified.HasValue 
						&& (SystemTime.UtcNow - sourceReplicationInformation.LastModified.Value).TotalMinutes < 10)
					{
						log.Info(string.Format("Replication source mismatch. Stored: {0}. Remote: {1}.", sourceReplicationInformation.Source, src));

						sourceReplicationInformation.LastAttachmentEtag = Etag.InvalidEtag;
						sourceReplicationInformation.LastDocumentEtag = Etag.InvalidEtag;
					}

					sourceReplicationInformation.ServerInstanceId = localServerInstanceId;
				}

				var maxNumberOfItemsToReceiveInSingleBatch = Database.Configuration.Replication.MaxNumberOfItemsToReceiveInSingleBatch;
				var availableMemory = MemoryStatistics.AvailableMemory;
				var lowMemory = availableMemory < 0.2 * MemoryStatistics.TotalPhysicalMemory && availableMemory < Database.Configuration.AvailableMemoryForRaisingBatchSizeLimit * 2;
				if (lowMemory)
				{
				    int size;
					var lastBatchSize = sourceReplicationInformation.LastBatchSize;
					if (lastBatchSize.HasValue && maxNumberOfItemsToReceiveInSingleBatch.HasValue)
                        size = Math.Min(lastBatchSize.Value, maxNumberOfItemsToReceiveInSingleBatch.Value);
					else if (lastBatchSize.HasValue)
                        size = lastBatchSize.Value;
					else if (maxNumberOfItemsToReceiveInSingleBatch.HasValue)
					    size = maxNumberOfItemsToReceiveInSingleBatch.Value;
					else
					    size = 128;

				    sourceReplicationInformation.MaxNumberOfItemsToReceiveInSingleBatch =
                        Math.Max(size / 2, 64);
				}
				else
				{
					sourceReplicationInformation.MaxNumberOfItemsToReceiveInSingleBatch = Database.Configuration.Replication.MaxNumberOfItemsToReceiveInSingleBatch;
				}

				var currentEtag = GetQueryStringValue("currentEtag");
				Log.Debug(() => string.Format("Got replication last etag request from {0}: [Local: {1} Remote: {2}]. LowMemory: {3}. MaxNumberOfItemsToReceiveInSingleBatch: {4}.", src, sourceReplicationInformation.LastDocumentEtag, currentEtag, lowMemory, sourceReplicationInformation.MaxNumberOfItemsToReceiveInSingleBatch));
				return GetMessageWithObject(sourceReplicationInformation);
			}
		}