public IEnumerable <JsonDocument> Handle(IEnumerable <JsonDocument> docs) { return(docs .Where(document => { var info = docActions.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(lastEtag) > 0) { if (Log.IsDebugEnabled) { Log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.TouchedEtag); } return false; } } string reason; return strategy.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) && prefetchingBehavior.FilterDocuments(document); })); }
public IEnumerable <JsonDocument> Handle(IEnumerable <JsonDocument> docs) { return(docs .Where(document => { string reason; return strategy.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) && prefetchingBehavior.FilterDocuments(document); })); }
public void ShouldHandleDelete() { Etag first = Etag.Empty.IncrementBy(1); prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(new[] { new JsonDocument { Etag = first, Key = "1" }, }); Etag second = Etag.Empty.IncrementBy(2); prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(new[] { new JsonDocument { Etag = second, Key = "2" }, }); prefetchingBehavior.AfterDelete("1", first); Assert.Equal(1, prefetchingBehavior.GetDocumentsBatchFrom(Etag.Empty).Count(x => prefetchingBehavior.FilterDocuments(x))); }
private void BackgroundSqlReplication() { int workCounter = 0; while (Database.WorkContext.DoWork) { var config = GetConfiguredReplicationDestinations(); if (config.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var localReplicationStatus = GetReplicationStatus(); var relevantConfigs = config.Where(x => { if (x.Disabled) { return(false); } var sqlReplicationStatistics = statistics.GetOrDefault(x.Name); if (sqlReplicationStatistics == null) { return(true); } return(SystemTime.UtcNow >= sqlReplicationStatistics.LastErrorTime); }) // have error or the timeout expired .ToList(); if (relevantConfigs.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus); if (leastReplicatedEtag == null) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var documents = prefetchingBehavior.GetDocumentsBatchFrom(leastReplicatedEtag); Etag latestEtag = null, lastBatchEtag = null; if (documents.Count != 0) { lastBatchEtag = documents[documents.Count - 1].Etag; } var replicationDuration = Stopwatch.StartNew(); documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here if (documents.Count != 0) { latestEtag = documents[documents.Count - 1].Etag; } documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false); var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >(); foreach (var relevantConfig in relevantConfigs) { var cfg = relevantConfig; Database.TransactionalStorage.Batch(accessor => { deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg), GetLastEtagFor(localReplicationStatus, cfg), latestEtag, 1024) .ToList(); }); } // No documents AND there aren't any deletes to replicate if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0) { if (latestEtag != null) { // so we filtered some documents, let us update the etag about that. foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags) { if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0) { lastReplicatedEtag.LastDocEtag = latestEtag; } } latestEtag = Etag.Max(latestEtag, lastBatchEtag); SaveNewReplicationStatus(localReplicationStatus, latestEtag); } else // no point in waiting if we just saved a new doc { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); } continue; } var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >(); try { BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig => { try { var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig); var deletedDocs = deletedDocsByConfig[replicationConfig]; var docsToReplicate = documents .Where(x => lastReplicatedEtag.CompareTo(x.Etag) <= 0) // haven't replicate the etag yet .Where(document => { var info = Database.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0) { log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, replicationConfig.Name, info.TouchedEtag); return(false); } } return(true); }) .ToList(); var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicate); if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) && ReplicateChangesToDestination(replicationConfig, docsToReplicate)) { if (deletedDocs.Count > 0) { Database.TransactionalStorage.Batch(accessor => accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag)); } successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag)); } } catch (Exception e) { log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Sql Replication failure to replication", Message = "Sql Replication could not replicate to " + replicationConfig.Name, UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name }); } }); if (successes.Count == 0) { continue; } foreach (var t in successes) { var cfg = t.Item1; var currentLatestEtag = t.Item2; var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase)); if (destEtag == null) { localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag { Name = cfg.Name, LastDocEtag = currentLatestEtag ?? Etag.Empty }); } else { destEtag.LastDocEtag = currentLatestEtag = currentLatestEtag ?? destEtag.LastDocEtag; } latestEtag = Etag.Max(latestEtag, currentLatestEtag); } latestEtag = Etag.Max(latestEtag, lastBatchEtag); SaveNewReplicationStatus(localReplicationStatus, latestEtag); } finally { AfterReplicationCompleted(successes.Count); var min = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray())); if (min != null) { var lastMinReplicatedEtag = min.ToEtag(); prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag); prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed); } } } }
private void BackgroundSqlReplication() { int workCounter = 0; while (Database.WorkContext.DoWork) { IsRunning = !shouldPause; if (!IsRunning) { continue; } var config = GetConfiguredReplicationDestinations(); if (config.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var localReplicationStatus = GetReplicationStatus(); var relevantConfigs = config.Where(x => { if (x.Disabled) { return(false); } var sqlReplicationStatistics = statistics.GetOrDefault(x.Name); if (sqlReplicationStatistics == null) { return(true); } return(SystemTime.UtcNow >= sqlReplicationStatistics.SuspendUntil); }) // have error or the timeout expired .ToList(); if (relevantConfigs.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus); if (leastReplicatedEtag == null) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } List <JsonDocument> documents; using (prefetchingBehavior.DocumentBatchFrom(leastReplicatedEtag, out documents)) { Etag latestEtag = null, lastBatchEtag = null; if (documents.Count != 0) { lastBatchEtag = documents[documents.Count - 1].Etag; } var replicationDuration = Stopwatch.StartNew(); documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here if (documents.Count != 0) { latestEtag = documents[documents.Count - 1].Etag; } documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false); var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >(); foreach (var relevantConfig in relevantConfigs) { var cfg = relevantConfig; Database.TransactionalStorage.Batch(accessor => { deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg), GetLastEtagFor(localReplicationStatus, cfg), latestEtag, MaxNumberOfDeletionsToReplicate + 1) .ToList(); }); } // No documents AND there aren't any deletes to replicate if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0) { if (latestEtag != null) { // so we filtered some documents, let us update the etag about that. foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags) { if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0) { lastReplicatedEtag.LastDocEtag = latestEtag; } } latestEtag = Etag.Max(latestEtag, lastBatchEtag); SaveNewReplicationStatus(localReplicationStatus); } else // no point in waiting if we just saved a new doc { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); } continue; } var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >(); try { var itemsToReplicate = documents.Select(x => { JsonDocument.EnsureIdInMetadata(x); var doc = x.ToJson(); doc[Constants.DocumentIdFieldName] = x.Key; return(new ReplicatedDoc { Document = doc, Etag = x.Etag, Key = x.Key, SerializedSizeOnDisk = x.SerializedSizeOnDisk }); }).ToList(); BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig => { try { var startTime = SystemTime.UtcNow; Stopwatch spRepTime = new Stopwatch(); spRepTime.Start(); var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig); var deletedDocs = deletedDocsByConfig[replicationConfig]; var docsToReplicate = itemsToReplicate .Where(x => lastReplicatedEtag.CompareTo(x.Etag) < 0) // haven't replicate the etag yet .Where(document => { var info = Database.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0) { log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, replicationConfig.Name, info.TouchedEtag); return(false); } } return(true); }); if (deletedDocs.Count >= MaxNumberOfDeletionsToReplicate + 1) { docsToReplicate = docsToReplicate.Where(x => EtagUtil.IsGreaterThan(x.Etag, deletedDocs[deletedDocs.Count - 1].Etag) == false); } var docsToReplicateAsList = docsToReplicate.ToList(); var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicateAsList); if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicateAsList.Count == 0) { currentLatestEtag = lastBatchEtag; } int countOfReplicatedItems = 0; if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) && ReplicateChangesToDestination(replicationConfig, docsToReplicateAsList, out countOfReplicatedItems)) { if (deletedDocs.Count > 0) { Database.TransactionalStorage.Batch(accessor => accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag)); } successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag)); } spRepTime.Stop(); var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick); var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig); sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds); UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicateAsList.Count); } catch (Exception e) { log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Sql Replication failure to replication", Message = "Sql Replication could not replicate to " + replicationConfig.Name, UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name }); } }); if (successes.Count == 0) { continue; } foreach (var t in successes) { var cfg = t.Item1; var currentLatestEtag = t.Item2; var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase)); if (destEtag == null) { localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag { Name = cfg.Name, LastDocEtag = currentLatestEtag ?? Etag.Empty }); } else { var lastDocEtag = destEtag.LastDocEtag; if (currentLatestEtag != null && EtagUtil.IsGreaterThan(currentLatestEtag, lastDocEtag)) { lastDocEtag = currentLatestEtag; } destEtag.LastDocEtag = lastDocEtag; } } SaveNewReplicationStatus(localReplicationStatus); } finally { AfterReplicationCompleted(successes.Count); var min = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray())); if (min != null) { var lastMinReplicatedEtag = min.ToEtag(); prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag); prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed); } } } } }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { var synchronizationEtag = etagSynchronizer.GetSynchronizationEtag(); var lastEtag = etagSynchronizer.CalculateSynchronizationEtag( synchronizationEtag, destinationsReplicationInformationForSource.LastDocumentEtag); int docsSinceLastReplEtag = 0; List <JsonDocument> docsToReplicate; List <JsonDocument> filteredDocsToReplicate; result.LastEtag = lastEtag; while (true) { docsToReplicate = GetDocsToReplicate(actions, result); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.GetRecentTouchesFor(document.Key); if (info != null) { if (info.PreTouchEtag.CompareTo(result.LastEtag) >= 0) { return(false); } } return(destination.FilterDocuments(destinationId, document.Key, document.Metadata) && prefetchingBehavior.FilterDocuments(document)); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag; if (lastDoc.LastModified.HasValue) { result.LastLastModified = lastDoc.LastModified.Value; } } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); } log.Debug(() => { if (docsSinceLastReplEtag == 0) { return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, lastEtag)); } if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) { return(string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, lastEtag, destination)); } var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), lastEtag)); }); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return(result); }