Ejemplo n.º 1
0
        public void ShouldDisableCollectingDocsAfterCommit()
        {
            Etag last = Etag.Empty;

            SystemTime.UtcDateTime = () => DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(15));

            prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(Enumerable.Range(0, 5).Select(x =>
            {
                last = EtagUtil.Increment(last, 1);

                return(new JsonDocument
                {
                    Etag = last,
                    Key = x.ToString(CultureInfo.InvariantCulture)
                });
            }).ToArray());

            last = EtagUtil.Increment(last, store.Configuration.MaxNumberOfItemsToProcessInSingleBatch);

            prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(Enumerable.Range(0, 5).Select(x =>
            {
                last = EtagUtil.Increment(last, 1);

                return(new JsonDocument
                {
                    Etag = last,
                    Key = x.ToString(CultureInfo.InvariantCulture)
                });
            }).ToArray());

            SystemTime.UtcDateTime = null;

            var documentsBatchFrom = prefetchingBehavior.GetDocumentsBatchFrom(Etag.Empty);

            prefetchingBehavior.CleanupDocuments(documentsBatchFrom.Last().Etag);

            Assert.True(prefetchingBehavior.DisableCollectingDocumentsAfterCommit);

            for (int i = 0; i < 5; i++)
            {
                last = EtagUtil.Increment(last, 1);
                prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(new[]
                {
                    new JsonDocument
                    {
                        Etag = last,
                        Key  = i.ToString(CultureInfo.InvariantCulture)
                    },
                });
            }

            Assert.Equal(0, prefetchingBehavior.InMemoryIndexingQueueSize);
        }
Ejemplo n.º 2
0
        private void BackgroundSqlReplication()
        {
            int workCounter = 0;

            while (Database.WorkContext.DoWork)
            {
                var config = GetConfiguredReplicationDestinations();
                if (config.Count == 0)
                {
                    Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
                    continue;
                }
                var localReplicationStatus = GetReplicationStatus();

                var relevantConfigs = config.Where(x =>
                {
                    if (x.Disabled)
                    {
                        return(false);
                    }
                    var sqlReplicationStatistics = statistics.GetOrDefault(x.Name);
                    if (sqlReplicationStatistics == null)
                    {
                        return(true);
                    }
                    return(SystemTime.UtcNow >= sqlReplicationStatistics.LastErrorTime);
                })                 // have error or the timeout expired
                                      .ToList();

                if (relevantConfigs.Count == 0)
                {
                    Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
                    continue;
                }

                var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus);

                if (leastReplicatedEtag == null)
                {
                    Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
                    continue;
                }

                var documents = prefetchingBehavior.GetDocumentsBatchFrom(leastReplicatedEtag);

                Etag latestEtag = null, lastBatchEtag = null;
                if (documents.Count != 0)
                {
                    lastBatchEtag = documents[documents.Count - 1].Etag;
                }

                var replicationDuration = Stopwatch.StartNew();
                documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase));                 // we ignore system documents here

                if (documents.Count != 0)
                {
                    latestEtag = documents[documents.Count - 1].Etag;
                }

                var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >();

                foreach (var relevantConfig in relevantConfigs)
                {
                    var cfg = relevantConfig;
                    Database.TransactionalStorage.Batch(accessor =>
                    {
                        deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg),
                                                                       GetLastEtagFor(localReplicationStatus, cfg),
                                                                       latestEtag,
                                                                       1024)
                                                   .ToList();
                    });
                }

                // No documents AND there aren't any deletes to replicate
                if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0)
                {
                    if (latestEtag != null)
                    {
                        // so we filtered some documents, let us update the etag about that.
                        foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags)
                        {
                            if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0)
                            {
                                lastReplicatedEtag.LastDocEtag = latestEtag;
                            }
                        }

                        latestEtag = Etag.Max(latestEtag, lastBatchEtag);
                        SaveNewReplicationStatus(localReplicationStatus, latestEtag);
                    }
                    else                     // no point in waiting if we just saved a new doc
                    {
                        Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
                    }
                    continue;
                }

                var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >();
                try
                {
                    BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig =>
                    {
                        try
                        {
                            var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig);

                            var deletedDocs     = deletedDocsByConfig[replicationConfig];
                            var docsToReplicate = documents
                                                  .Where(x => lastReplicatedEtag.CompareTo(x.Etag) <= 0)               // haven't replicate the etag yet
                                                  .ToList();

                            var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicate);

                            if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) &&
                                ReplicateChangesToDesintation(replicationConfig, docsToReplicate))
                            {
                                if (deletedDocs.Count > 0)
                                {
                                    Database.TransactionalStorage.Batch(accessor =>
                                                                        accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag));
                                }
                                successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag));
                            }
                        }
                        catch (Exception e)
                        {
                            log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
                            Database.AddAlert(new Alert
                            {
                                AlertLevel = AlertLevel.Error,
                                CreatedAt  = SystemTime.UtcNow,
                                Exception  = e.ToString(),
                                Title      = "Sql Replication failure to replication",
                                Message    = "Sql Replication could not replicate to " + replicationConfig.Name,
                                UniqueKey  = "Sql Replication could not replicate to " + replicationConfig.Name
                            });
                        }
                    });
                    if (successes.Count == 0)
                    {
                        continue;
                    }
                    foreach (var t in successes)
                    {
                        var cfg = t.Item1;
                        var currentLatestEtag = t.Item2;
                        var destEtag          = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
                        if (destEtag == null)
                        {
                            localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
                            {
                                Name        = cfg.Name,
                                LastDocEtag = currentLatestEtag ?? Etag.Empty
                            });
                        }
                        else
                        {
                            destEtag.LastDocEtag = currentLatestEtag = currentLatestEtag ?? destEtag.LastDocEtag;
                        }
                        latestEtag = Etag.Max(latestEtag, currentLatestEtag);
                    }

                    latestEtag = Etag.Max(latestEtag, lastBatchEtag);
                    SaveNewReplicationStatus(localReplicationStatus, latestEtag);
                }
                finally
                {
                    AfterReplicationCompleted(successes.Count);
                    var lastMinReplicatedEtag = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray())).ToEtag();
                    prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag);
                    prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed);
                }
            }
        }
Ejemplo n.º 3
0
        private void BackgroundSqlReplication()
        {
            int workCounter = 0;

            while (Database.WorkContext.DoWork)
            {
                IsRunning = !shouldPause;

                if (!IsRunning)
                {
                    continue;
                }

                var config = GetConfiguredReplicationDestinations();
                if (config.Count == 0)
                {
                    Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
                    continue;
                }
                var localReplicationStatus = GetReplicationStatus();

                var relevantConfigs = config.Where(x =>
                {
                    if (x.Disabled)
                    {
                        return(false);
                    }
                    var sqlReplicationStatistics = statistics.GetOrDefault(x.Name);
                    if (sqlReplicationStatistics == null)
                    {
                        return(true);
                    }
                    return(SystemTime.UtcNow >= sqlReplicationStatistics.SuspendUntil);
                })                 // have error or the timeout expired
                                      .ToList();

                if (relevantConfigs.Count == 0)
                {
                    Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
                    continue;
                }

                var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus);

                if (leastReplicatedEtag == null)
                {
                    Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
                    continue;
                }

                List <JsonDocument> documents;

                using (prefetchingBehavior.DocumentBatchFrom(leastReplicatedEtag, out documents))
                {
                    Etag latestEtag = null, lastBatchEtag = null;
                    if (documents.Count != 0)
                    {
                        lastBatchEtag = documents[documents.Count - 1].Etag;
                    }

                    var replicationDuration = Stopwatch.StartNew();
                    documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase));                     // we ignore system documents here

                    if (documents.Count != 0)
                    {
                        latestEtag = documents[documents.Count - 1].Etag;
                    }

                    documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false);

                    var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >();

                    foreach (var relevantConfig in relevantConfigs)
                    {
                        var cfg = relevantConfig;
                        Database.TransactionalStorage.Batch(accessor =>
                        {
                            deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg),
                                                                           GetLastEtagFor(localReplicationStatus, cfg),
                                                                           latestEtag,
                                                                           MaxNumberOfDeletionsToReplicate + 1)
                                                       .ToList();
                        });
                    }

                    // No documents AND there aren't any deletes to replicate
                    if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0)
                    {
                        if (latestEtag != null)
                        {
                            // so we filtered some documents, let us update the etag about that.
                            foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags)
                            {
                                if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0)
                                {
                                    lastReplicatedEtag.LastDocEtag = latestEtag;
                                }
                            }

                            latestEtag = Etag.Max(latestEtag, lastBatchEtag);
                            SaveNewReplicationStatus(localReplicationStatus);
                        }
                        else                         // no point in waiting if we just saved a new doc
                        {
                            Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
                        }
                        continue;
                    }

                    var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >();
                    try
                    {
                        var itemsToReplicate = documents.Select(x =>
                        {
                            JsonDocument.EnsureIdInMetadata(x);
                            var doc = x.ToJson();
                            doc[Constants.DocumentIdFieldName] = x.Key;

                            return(new ReplicatedDoc
                            {
                                Document = doc,
                                Etag = x.Etag,
                                Key = x.Key,
                                SerializedSizeOnDisk = x.SerializedSizeOnDisk
                            });
                        }).ToList();

                        BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig =>
                        {
                            try
                            {
                                var startTime       = SystemTime.UtcNow;
                                Stopwatch spRepTime = new Stopwatch();
                                spRepTime.Start();
                                var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig);

                                var deletedDocs     = deletedDocsByConfig[replicationConfig];
                                var docsToReplicate = itemsToReplicate
                                                      .Where(x => lastReplicatedEtag.CompareTo(x.Etag) < 0)                   // haven't replicate the etag yet
                                                      .Where(document =>
                                {
                                    var info = Database.Documents.GetRecentTouchesFor(document.Key);
                                    if (info != null)
                                    {
                                        if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0)
                                        {
                                            log.Debug(
                                                "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
                                                document.Key, replicationConfig.Name, info.TouchedEtag);
                                            return(false);
                                        }
                                    }
                                    return(true);
                                });

                                if (deletedDocs.Count >= MaxNumberOfDeletionsToReplicate + 1)
                                {
                                    docsToReplicate = docsToReplicate.Where(x => EtagUtil.IsGreaterThan(x.Etag, deletedDocs[deletedDocs.Count - 1].Etag) == false);
                                }

                                var docsToReplicateAsList = docsToReplicate.ToList();

                                var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicateAsList);
                                if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicateAsList.Count == 0)
                                {
                                    currentLatestEtag = lastBatchEtag;
                                }

                                int countOfReplicatedItems = 0;
                                if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) &&
                                    ReplicateChangesToDestination(replicationConfig, docsToReplicateAsList, out countOfReplicatedItems))
                                {
                                    if (deletedDocs.Count > 0)
                                    {
                                        Database.TransactionalStorage.Batch(accessor =>
                                                                            accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag));
                                    }
                                    successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag));
                                }

                                spRepTime.Stop();
                                var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick);

                                var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig);
                                sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems);
                                sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems);
                                sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds);

                                UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicateAsList.Count);
                            }
                            catch (Exception e)
                            {
                                log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
                                Database.AddAlert(new Alert
                                {
                                    AlertLevel = AlertLevel.Error,
                                    CreatedAt  = SystemTime.UtcNow,
                                    Exception  = e.ToString(),
                                    Title      = "Sql Replication failure to replication",
                                    Message    = "Sql Replication could not replicate to " + replicationConfig.Name,
                                    UniqueKey  = "Sql Replication could not replicate to " + replicationConfig.Name
                                });
                            }
                        });
                        if (successes.Count == 0)
                        {
                            continue;
                        }
                        foreach (var t in successes)
                        {
                            var cfg = t.Item1;
                            var currentLatestEtag = t.Item2;
                            var destEtag          = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
                            if (destEtag == null)
                            {
                                localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
                                {
                                    Name        = cfg.Name,
                                    LastDocEtag = currentLatestEtag ?? Etag.Empty
                                });
                            }
                            else
                            {
                                var lastDocEtag = destEtag.LastDocEtag;
                                if (currentLatestEtag != null && EtagUtil.IsGreaterThan(currentLatestEtag, lastDocEtag))
                                {
                                    lastDocEtag = currentLatestEtag;
                                }

                                destEtag.LastDocEtag = lastDocEtag;
                            }
                        }

                        SaveNewReplicationStatus(localReplicationStatus);
                    }
                    finally
                    {
                        AfterReplicationCompleted(successes.Count);
                        var min = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray()));
                        if (min != null)
                        {
                            var lastMinReplicatedEtag = min.ToEtag();
                            prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag);
                            prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed);
                        }
                    }
                }
            }
        }
Ejemplo n.º 4
0
        private void Execute()
        {
            using (LogContext.WithDatabase(docDb.Name))
            {
                var name = GetType().Name;

                var  timeToWaitInMinutes = TimeSpan.FromMinutes(5);
                bool runningBecauseOfDataModifications = false;
                var  context = docDb.WorkContext;
                NotifySiblings();
                while (context.DoWork)
                {
                    try
                    {
                        using (docDb.DisableAllTriggersForCurrentThread())
                        {
                            var destinations = GetReplicationDestinations();

                            if (destinations.Length == 0)
                            {
                                WarnIfNoReplicationTargetsWereFound();
                            }
                            else
                            {
                                var currentReplicationAttempts = Interlocked.Increment(ref replicationAttempts);

                                var copyOfrunningBecauseOfDataModifications = runningBecauseOfDataModifications;
                                var destinationForReplication = destinations
                                                                .Where(dest =>
                                {
                                    if (copyOfrunningBecauseOfDataModifications == false)
                                    {
                                        return(true);
                                    }
                                    return(IsNotFailing(dest, currentReplicationAttempts));
                                });

                                var startedTasks = new List <Task>();

                                foreach (var dest in destinationForReplication)
                                {
                                    var destination = dest;
                                    var holder      = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, new IntHolder());
                                    if (Thread.VolatileRead(ref holder.Value) == 1)
                                    {
                                        continue;
                                    }
                                    Thread.VolatileWrite(ref holder.Value, 1);
                                    var replicationTask = Task.Factory.StartNew(() =>
                                    {
                                        using (LogContext.WithDatabase(docDb.Name))
                                        {
                                            try
                                            {
                                                if (ReplicateTo(destination))
                                                {
                                                    docDb.WorkContext.NotifyAboutWork();
                                                }
                                            }
                                            catch (Exception e)
                                            {
                                                log.ErrorException("Could not replicate to " + destination, e);
                                            }
                                        }
                                    });

                                    startedTasks.Add(replicationTask);

                                    activeTasks.Enqueue(replicationTask);
                                    replicationTask.ContinueWith(_ =>
                                    {
                                        // here we purge all the completed tasks at the head of the queue
                                        Task task;
                                        while (activeTasks.TryPeek(out task))
                                        {
                                            if (!task.IsCompleted && !task.IsCanceled && !task.IsFaulted)
                                            {
                                                break;
                                            }
                                            activeTasks.TryDequeue(out task);                                             // remove it from end
                                        }
                                    });
                                }

#if NET45
                                Task
#else
                                TaskEx
#endif
                                .WhenAll(startedTasks.ToArray()).ContinueWith(t =>
                                {
                                    if (destinationStats.Count != 0)
                                    {
                                        var minLastReplicatedEtag = destinationStats.Where(x => x.Value.LastReplicatedEtag != null)
                                                                    .Select(x => x.Value.LastReplicatedEtag)
                                                                    .Min(x => new ComparableByteArray(x.ToByteArray()));

                                        if (minLastReplicatedEtag != null)
                                        {
                                            prefetchingBehavior.CleanupDocuments(minLastReplicatedEtag.ToEtag());
                                        }
                                    }
                                }).AssertNotFailed();
                            }
                        }
                    }
                    catch (Exception e)
                    {
                        log.ErrorException("Failed to perform replication", e);
                    }

                    runningBecauseOfDataModifications = context.WaitForWork(timeToWaitInMinutes, ref workCounter, name);
                    timeToWaitInMinutes = runningBecauseOfDataModifications
                                                                                        ? TimeSpan.FromSeconds(30)
                                                                                        : TimeSpan.FromMinutes(5);
                }
            }
        }