private void SetPrefetcherForIndexingGroup(SqlConfigGroup sqlConfig, ConcurrentSet <PrefetchingBehavior> usedPrefetchers) { sqlConfig.PrefetchingBehavior = TryGetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers) ?? TryGetDefaultPrefetcher(sqlConfig.LastReplicatedEtag, usedPrefetchers) ?? GetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers); sqlConfig.PrefetchingBehavior.AdditionalInfo = string.Format("Default prefetcher: {0}. For sql config group: [Configs: {1}, LastReplicatedEtag: {2}]", sqlConfig.PrefetchingBehavior == defaultPrefetchingBehavior, string.Join(", ", sqlConfig.ConfigsToWorkOn.Select(y => y.Name)), sqlConfig.LastReplicatedEtag); }
private void SetPrefetcherForIndexingGroup(SqlConfigGroup sqlConfig, ConcurrentSet <PrefetchingBehavior> usedPrefetchers) { var entityNames = new HashSet <string>(sqlConfig.ConfigsToWorkOn.Select(x => x.RavenEntityName), StringComparer.OrdinalIgnoreCase); sqlConfig.PrefetchingBehavior = TryGetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers, entityNames) ?? GetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers, entityNames); sqlConfig.PrefetchingBehavior.SetEntityNames(entityNames); sqlConfig.PrefetchingBehavior.AdditionalInfo = $"For SQL config group: [Configs: {string.Join(", ", sqlConfig.ConfigsToWorkOn.Select(y => y.Name))}, " + $"Last Replicated Etag: {sqlConfig.LastReplicatedEtag}], collections: {string.Join(", ", entityNames)}"; }
private void SetPrefetcherForIndexingGroup(SqlConfigGroup sqlConfig, ConcurrentSet <PrefetchingBehavior> usedPrefetchers) { var entityNames = new HashSet <string>(sqlConfig.ConfigsToWorkOn.Select(x => x.RavenEntityName), StringComparer.OrdinalIgnoreCase); sqlConfig.PrefetchingBehavior = TryGetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers, entityNames) ?? TryGetDefaultPrefetcher(sqlConfig.LastReplicatedEtag, usedPrefetchers) ?? GetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers); sqlConfig.PrefetchingBehavior.AdditionalInfo = string.Format("Default prefetcher: {0}. For sql config group: [Configs: {1}, LastReplicatedEtag: {2}]", sqlConfig.PrefetchingBehavior == defaultPrefetchingBehavior, string.Join(", ", sqlConfig.ConfigsToWorkOn.Select(y => y.Name)), sqlConfig.LastReplicatedEtag); }
private void BackgroundSqlReplication() { int workCounter = 0; while (Database.WorkContext.DoWork) { IsRunning = !IsHotSpare() && !shouldPause; if (!IsRunning) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var config = GetConfiguredReplicationDestinations(); if (config.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var localReplicationStatus = GetReplicationStatus(); // remove all last replicated statuses which are not in the config UpdateLastReplicatedStatus(localReplicationStatus, config); var relevantConfigs = config.Where(x => { if (x.Disabled) { return(false); } var sqlReplicationStatistics = statistics.GetOrDefault(x.Name); if (sqlReplicationStatistics == null) { return(true); } return(SystemTime.UtcNow >= sqlReplicationStatistics.SuspendUntil); }) // have error or the timeout expired .ToList(); var configGroups = SqlReplicationClassifier.GroupConfigs(relevantConfigs, c => GetLastEtagFor(localReplicationStatus, c)); if (configGroups.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var usedPrefetchers = new ConcurrentSet <PrefetchingBehavior>(); var groupedConfigs = configGroups .Select(x => { var result = new SqlConfigGroup { LastReplicatedEtag = x.Key, ConfigsToWorkOn = x.Value }; SetPrefetcherForIndexingGroup(result, usedPrefetchers); return(result); }) .ToList(); var successes = new ConcurrentQueue <Tuple <SqlReplicationConfigWithLastReplicatedEtag, Etag> >(); var waitForWork = new bool[groupedConfigs.Count]; try { BackgroundTaskExecuter.Instance.ExecuteAll(Database.WorkContext, groupedConfigs, (sqlConfigGroup, i) => { Database.WorkContext.CancellationToken.ThrowIfCancellationRequested(); var prefetchingBehavior = sqlConfigGroup.PrefetchingBehavior; var configsToWorkOn = sqlConfigGroup.ConfigsToWorkOn; List <JsonDocument> documents; var entityNamesToIndex = new HashSet <string>(configsToWorkOn.Select(x => x.RavenEntityName), StringComparer.OrdinalIgnoreCase); using (prefetchingBehavior.DocumentBatchFrom(sqlConfigGroup.LastReplicatedEtag, out documents, entityNamesToIndex)) { Etag latestEtag = null, lastBatchEtag = null; if (documents.Count != 0) { lastBatchEtag = documents[documents.Count - 1].Etag; } var replicationDuration = Stopwatch.StartNew(); documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here if (documents.Count != 0) { latestEtag = documents[documents.Count - 1].Etag; } documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false); var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >(); foreach (var configToWorkOn in configsToWorkOn) { var cfg = configToWorkOn; Database.TransactionalStorage.Batch(accessor => { deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg), cfg.LastReplicatedEtag, latestEtag, MaxNumberOfDeletionsToReplicate + 1) .ToList(); }); } // No documents AND there aren't any deletes to replicate if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0) { // so we filtered some documents, let us update the etag about that. if (latestEtag != null) { foreach (var configToWorkOn in configsToWorkOn) { successes.Enqueue(Tuple.Create(configToWorkOn, latestEtag)); } } else { waitForWork[i] = true; } return; } var itemsToReplicate = documents.Select(x => { JsonDocument.EnsureIdInMetadata(x); var doc = x.ToJson(); doc[Constants.DocumentIdFieldName] = x.Key; return(new ReplicatedDoc { Document = doc, Etag = x.Etag, Key = x.Key, SerializedSizeOnDisk = x.SerializedSizeOnDisk }); }).ToList(); try { BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, configsToWorkOn, replicationConfig => { try { var startTime = SystemTime.UtcNow; var spRepTime = new Stopwatch(); spRepTime.Start(); var lastReplicatedEtag = replicationConfig.LastReplicatedEtag; var deletedDocs = deletedDocsByConfig[replicationConfig]; var docsToReplicate = itemsToReplicate .Where(x => lastReplicatedEtag.CompareTo(x.Etag) < 0) // haven't replicate the etag yet .Where(document => { var info = Database.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0) { if (Log.IsDebugEnabled) { Log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, replicationConfig.Name, info.TouchedEtag); } return(false); } } return(true); }); if (deletedDocs.Count >= MaxNumberOfDeletionsToReplicate + 1) { docsToReplicate = docsToReplicate.Where(x => EtagUtil.IsGreaterThan(x.Etag, deletedDocs[deletedDocs.Count - 1].Etag) == false); } var docsToReplicateAsList = docsToReplicate.ToList(); var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicateAsList); if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicateAsList.Count == 0) { currentLatestEtag = lastBatchEtag; } int countOfReplicatedItems = 0; if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) && ReplicateChangesToDestination(replicationConfig, docsToReplicateAsList, out countOfReplicatedItems)) { if (deletedDocs.Count > 0) { Database.TransactionalStorage.Batch(accessor => accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag)); } successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag)); } spRepTime.Stop(); var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick); var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig); sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds); UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicateAsList.Count); } catch (Exception e) { Log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Sql Replication failure to replication", Message = "Sql Replication could not replicate to " + replicationConfig.Name, UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name }); } }); } finally { prefetchingBehavior.CleanupDocuments(lastBatchEtag); prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed); } } }); if (successes.Count == 0) { if (waitForWork.All(x => x)) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); } continue; } foreach (var t in successes) { var cfg = t.Item1; var currentLatestEtag = t.Item2; //If a reset was requested we don't want to update the last replicated etag. //If we do register the success the reset will become a noop. bool isReset; if (ResetRequested.TryGetValue(t.Item1.Name, out isReset) && isReset) { continue; } var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase)); if (destEtag == null) { localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag { Name = cfg.Name, LastDocEtag = currentLatestEtag ?? Etag.Empty }); } else { var lastDocEtag = destEtag.LastDocEtag; if (currentLatestEtag != null && EtagUtil.IsGreaterThan(currentLatestEtag, lastDocEtag)) { lastDocEtag = currentLatestEtag; } destEtag.LastDocEtag = lastDocEtag; } } //We are done recording success for this batch so we can clear the reset dictionary ResetRequested.Clear(); SaveNewReplicationStatus(localReplicationStatus); } finally { AfterReplicationCompleted(successes.Count); RemoveUnusedPrefetchers(usedPrefetchers); } } }
private void BackgroundSqlReplication() { int workCounter = 0; while (Database.WorkContext.DoWork) { IsRunning = !shouldPause; if (!IsRunning) continue; var config = GetConfiguredReplicationDestinations(); if (config.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var localReplicationStatus = GetReplicationStatus(); var relevantConfigs = config.Where(x => { if (x.Disabled) return false; var sqlReplicationStatistics = statistics.GetOrDefault(x.Name); if (sqlReplicationStatistics == null) return true; return SystemTime.UtcNow >= sqlReplicationStatistics.SuspendUntil; }) // have error or the timeout expired .ToList(); var configGroups = SqlReplicationClassifier.GroupConfigs(relevantConfigs, c => GetLastEtagFor(localReplicationStatus, c)); if (configGroups.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var usedPrefetchers = new ConcurrentSet<PrefetchingBehavior>(); var groupedConfigs = configGroups .Select(x => { var result = new SqlConfigGroup { LastReplicatedEtag = x.Key, ConfigsToWorkOn = x.Value, PrefetchingBehavior = GetPrefetcherFor(x.Key, usedPrefetchers) }; result.PrefetchingBehavior.AdditionalInfo = string.Format("Default prefetcher: {0}. For sql config group: [Configs: {1}, LastReplicatedEtag: {2}]", result.PrefetchingBehavior == defaultPrefetchingBehavior, string.Join(", ", result.ConfigsToWorkOn.Select(y => y.Name)), result.LastReplicatedEtag); return result; }) .ToList(); var successes = new ConcurrentQueue<Tuple<SqlReplicationConfigWithLastReplicatedEtag, Etag>>(); var waitForWork = new bool[groupedConfigs.Count]; try { BackgroundTaskExecuter.Instance.ExecuteAll(Database.WorkContext, groupedConfigs, (sqlConfigGroup, i) => { Database.WorkContext.CancellationToken.ThrowIfCancellationRequested(); var prefetchingBehavior = sqlConfigGroup.PrefetchingBehavior; var configsToWorkOn = sqlConfigGroup.ConfigsToWorkOn; List<JsonDocument> documents; using (prefetchingBehavior.DocumentBatchFrom(sqlConfigGroup.LastReplicatedEtag, out documents)) { Etag latestEtag = null, lastBatchEtag = null; if (documents.Count != 0) lastBatchEtag = documents[documents.Count - 1].Etag; var replicationDuration = Stopwatch.StartNew(); documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here if (documents.Count != 0) latestEtag = documents[documents.Count - 1].Etag; documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false); var deletedDocsByConfig = new Dictionary<SqlReplicationConfig, List<ListItem>>(); foreach (var configToWorkOn in configsToWorkOn) { var cfg = configToWorkOn; Database.TransactionalStorage.Batch(accessor => { deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg), cfg.LastReplicatedEtag, latestEtag, MaxNumberOfDeletionsToReplicate + 1) .ToList(); }); } // No documents AND there aren't any deletes to replicate if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0) { // so we filtered some documents, let us update the etag about that. if (latestEtag != null) { foreach (var configToWorkOn in configsToWorkOn) successes.Enqueue(Tuple.Create(configToWorkOn, latestEtag)); } else { waitForWork[i] = true; } return; } var itemsToReplicate = documents.Select(x => { JsonDocument.EnsureIdInMetadata(x); var doc = x.ToJson(); doc[Constants.DocumentIdFieldName] = x.Key; return new ReplicatedDoc { Document = doc, Etag = x.Etag, Key = x.Key, SerializedSizeOnDisk = x.SerializedSizeOnDisk }; }).ToList(); try { BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, configsToWorkOn, replicationConfig => { try { var startTime = SystemTime.UtcNow; var spRepTime = new Stopwatch(); spRepTime.Start(); var lastReplicatedEtag = replicationConfig.LastReplicatedEtag; var deletedDocs = deletedDocsByConfig[replicationConfig]; var docsToReplicate = itemsToReplicate .Where(x => lastReplicatedEtag.CompareTo(x.Etag) < 0) // haven't replicate the etag yet .Where(document => { var info = Database.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0) { Log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, replicationConfig.Name, info.TouchedEtag); return false; } } return true; }); if (deletedDocs.Count >= MaxNumberOfDeletionsToReplicate + 1) docsToReplicate = docsToReplicate.Where(x => EtagUtil.IsGreaterThan(x.Etag, deletedDocs[deletedDocs.Count - 1].Etag) == false); var docsToReplicateAsList = docsToReplicate.ToList(); var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicateAsList); if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicateAsList.Count == 0) currentLatestEtag = lastBatchEtag; int countOfReplicatedItems = 0; if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) && ReplicateChangesToDestination(replicationConfig, docsToReplicateAsList, out countOfReplicatedItems)) { if (deletedDocs.Count > 0) { Database.TransactionalStorage.Batch(accessor => accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag)); } successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag)); } spRepTime.Stop(); var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick); var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig); sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds); UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicateAsList.Count); } catch (Exception e) { Log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Sql Replication failure to replication", Message = "Sql Replication could not replicate to " + replicationConfig.Name, UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name }); } }); } finally { prefetchingBehavior.CleanupDocuments(lastBatchEtag); prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed); } } }); if (successes.Count == 0) { if (waitForWork.All(x => x)) Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } foreach (var t in successes) { var cfg = t.Item1; var currentLatestEtag = t.Item2; var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase)); if (destEtag == null) { localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag { Name = cfg.Name, LastDocEtag = currentLatestEtag ?? Etag.Empty }); } else { var lastDocEtag = destEtag.LastDocEtag; if (currentLatestEtag != null && EtagUtil.IsGreaterThan(currentLatestEtag, lastDocEtag)) lastDocEtag = currentLatestEtag; destEtag.LastDocEtag = lastDocEtag; } } SaveNewReplicationStatus(localReplicationStatus); } finally { AfterReplicationCompleted(successes.Count); RemoveUnusedPrefetchers(usedPrefetchers); } } }
private void SetPrefetcherForIndexingGroup(SqlConfigGroup sqlConfig, ConcurrentSet<PrefetchingBehavior> usedPrefetchers) { sqlConfig.PrefetchingBehavior = TryGetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers) ?? TryGetDefaultPrefetcher(sqlConfig.LastReplicatedEtag, usedPrefetchers) ?? GetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers); sqlConfig.PrefetchingBehavior.AdditionalInfo = string.Format("Default prefetcher: {0}. For sql config group: [Configs: {1}, LastReplicatedEtag: {2}]", sqlConfig.PrefetchingBehavior == defaultPrefetchingBehavior, string.Join(", ", sqlConfig.ConfigsToWorkOn.Select(y => y.Name)), sqlConfig.LastReplicatedEtag); }