private void SetPrefetcherForIndexingGroup(SqlConfigGroup sqlConfig, ConcurrentSet <PrefetchingBehavior> usedPrefetchers)
{
sqlConfig.PrefetchingBehavior = TryGetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers) ??
TryGetDefaultPrefetcher(sqlConfig.LastReplicatedEtag, usedPrefetchers) ??
GetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers);
sqlConfig.PrefetchingBehavior.AdditionalInfo =
string.Format("Default prefetcher: {0}. For sql config group: [Configs: {1}, LastReplicatedEtag: {2}]",
sqlConfig.PrefetchingBehavior == defaultPrefetchingBehavior,
string.Join(", ", sqlConfig.ConfigsToWorkOn.Select(y => y.Name)),
sqlConfig.LastReplicatedEtag);
}
private void SetPrefetcherForIndexingGroup(SqlConfigGroup sqlConfig, ConcurrentSet <PrefetchingBehavior> usedPrefetchers)
{
var entityNames = new HashSet <string>(sqlConfig.ConfigsToWorkOn.Select(x => x.RavenEntityName), StringComparer.OrdinalIgnoreCase);
sqlConfig.PrefetchingBehavior = TryGetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers, entityNames) ??
GetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers, entityNames);
sqlConfig.PrefetchingBehavior.SetEntityNames(entityNames);
sqlConfig.PrefetchingBehavior.AdditionalInfo =
$"For SQL config group: [Configs: {string.Join(", ", sqlConfig.ConfigsToWorkOn.Select(y => y.Name))}, " +
$"Last Replicated Etag: {sqlConfig.LastReplicatedEtag}], collections: {string.Join(", ", entityNames)}";
}
private void SetPrefetcherForIndexingGroup(SqlConfigGroup sqlConfig, ConcurrentSet <PrefetchingBehavior> usedPrefetchers)
{
var entityNames = new HashSet <string>(sqlConfig.ConfigsToWorkOn.Select(x => x.RavenEntityName), StringComparer.OrdinalIgnoreCase);
sqlConfig.PrefetchingBehavior = TryGetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers, entityNames) ??
TryGetDefaultPrefetcher(sqlConfig.LastReplicatedEtag, usedPrefetchers) ??
GetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers);
sqlConfig.PrefetchingBehavior.AdditionalInfo =
string.Format("Default prefetcher: {0}. For sql config group: [Configs: {1}, LastReplicatedEtag: {2}]",
sqlConfig.PrefetchingBehavior == defaultPrefetchingBehavior,
string.Join(", ", sqlConfig.ConfigsToWorkOn.Select(y => y.Name)),
sqlConfig.LastReplicatedEtag);
}
private void BackgroundSqlReplication()
{
int workCounter = 0;
while (Database.WorkContext.DoWork)
{
IsRunning = !IsHotSpare() && !shouldPause;
if (!IsRunning)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var config = GetConfiguredReplicationDestinations();
if (config.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var localReplicationStatus = GetReplicationStatus();
// remove all last replicated statuses which are not in the config
UpdateLastReplicatedStatus(localReplicationStatus, config);
var relevantConfigs = config.Where(x =>
{
if (x.Disabled)
{
return(false);
}
var sqlReplicationStatistics = statistics.GetOrDefault(x.Name);
if (sqlReplicationStatistics == null)
{
return(true);
}
return(SystemTime.UtcNow >= sqlReplicationStatistics.SuspendUntil);
}) // have error or the timeout expired
.ToList();
var configGroups = SqlReplicationClassifier.GroupConfigs(relevantConfigs, c => GetLastEtagFor(localReplicationStatus, c));
if (configGroups.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var usedPrefetchers = new ConcurrentSet <PrefetchingBehavior>();
var groupedConfigs = configGroups
.Select(x =>
{
var result = new SqlConfigGroup
{
LastReplicatedEtag = x.Key,
ConfigsToWorkOn = x.Value
};
SetPrefetcherForIndexingGroup(result, usedPrefetchers);
return(result);
})
.ToList();
var successes = new ConcurrentQueue <Tuple <SqlReplicationConfigWithLastReplicatedEtag, Etag> >();
var waitForWork = new bool[groupedConfigs.Count];
try
{
BackgroundTaskExecuter.Instance.ExecuteAll(Database.WorkContext, groupedConfigs, (sqlConfigGroup, i) =>
{
Database.WorkContext.CancellationToken.ThrowIfCancellationRequested();
var prefetchingBehavior = sqlConfigGroup.PrefetchingBehavior;
var configsToWorkOn = sqlConfigGroup.ConfigsToWorkOn;
List <JsonDocument> documents;
var entityNamesToIndex = new HashSet <string>(configsToWorkOn.Select(x => x.RavenEntityName), StringComparer.OrdinalIgnoreCase);
using (prefetchingBehavior.DocumentBatchFrom(sqlConfigGroup.LastReplicatedEtag, out documents, entityNamesToIndex))
{
Etag latestEtag = null, lastBatchEtag = null;
if (documents.Count != 0)
{
lastBatchEtag = documents[documents.Count - 1].Etag;
}
var replicationDuration = Stopwatch.StartNew();
documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here
if (documents.Count != 0)
{
latestEtag = documents[documents.Count - 1].Etag;
}
documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false);
var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >();
foreach (var configToWorkOn in configsToWorkOn)
{
var cfg = configToWorkOn;
Database.TransactionalStorage.Batch(accessor =>
{
deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg),
cfg.LastReplicatedEtag,
latestEtag,
MaxNumberOfDeletionsToReplicate + 1)
.ToList();
});
}
// No documents AND there aren't any deletes to replicate
if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0)
{
// so we filtered some documents, let us update the etag about that.
if (latestEtag != null)
{
foreach (var configToWorkOn in configsToWorkOn)
{
successes.Enqueue(Tuple.Create(configToWorkOn, latestEtag));
}
}
else
{
waitForWork[i] = true;
}
return;
}
var itemsToReplicate = documents.Select(x =>
{
JsonDocument.EnsureIdInMetadata(x);
var doc = x.ToJson();
doc[Constants.DocumentIdFieldName] = x.Key;
return(new ReplicatedDoc
{
Document = doc,
Etag = x.Etag,
Key = x.Key,
SerializedSizeOnDisk = x.SerializedSizeOnDisk
});
}).ToList();
try
{
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, configsToWorkOn, replicationConfig =>
{
try
{
var startTime = SystemTime.UtcNow;
var spRepTime = new Stopwatch();
spRepTime.Start();
var lastReplicatedEtag = replicationConfig.LastReplicatedEtag;
var deletedDocs = deletedDocsByConfig[replicationConfig];
var docsToReplicate = itemsToReplicate
.Where(x => lastReplicatedEtag.CompareTo(x.Etag) < 0) // haven't replicate the etag yet
.Where(document =>
{
var info = Database.Documents.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0)
{
if (Log.IsDebugEnabled)
{
Log.Debug(
"Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
document.Key, replicationConfig.Name, info.TouchedEtag);
}
return(false);
}
}
return(true);
});
if (deletedDocs.Count >= MaxNumberOfDeletionsToReplicate + 1)
{
docsToReplicate = docsToReplicate.Where(x => EtagUtil.IsGreaterThan(x.Etag, deletedDocs[deletedDocs.Count - 1].Etag) == false);
}
var docsToReplicateAsList = docsToReplicate.ToList();
var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicateAsList);
if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicateAsList.Count == 0)
{
currentLatestEtag = lastBatchEtag;
}
int countOfReplicatedItems = 0;
if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) &&
ReplicateChangesToDestination(replicationConfig, docsToReplicateAsList, out countOfReplicatedItems))
{
if (deletedDocs.Count > 0)
{
Database.TransactionalStorage.Batch(accessor =>
accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag));
}
successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag));
}
spRepTime.Stop();
var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick);
var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig);
sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems);
sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems);
sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds);
UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicateAsList.Count);
}
catch (Exception e)
{
Log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Exception = e.ToString(),
Title = "Sql Replication failure to replication",
Message = "Sql Replication could not replicate to " + replicationConfig.Name,
UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name
});
}
});
}
finally
{
prefetchingBehavior.CleanupDocuments(lastBatchEtag);
prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed);
}
}
});
if (successes.Count == 0)
{
if (waitForWork.All(x => x))
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
}
continue;
}
foreach (var t in successes)
{
var cfg = t.Item1;
var currentLatestEtag = t.Item2;
//If a reset was requested we don't want to update the last replicated etag.
//If we do register the success the reset will become a noop.
bool isReset;
if (ResetRequested.TryGetValue(t.Item1.Name, out isReset) && isReset)
{
continue;
}
var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
if (destEtag == null)
{
localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
{
Name = cfg.Name,
LastDocEtag = currentLatestEtag ?? Etag.Empty
});
}
else
{
var lastDocEtag = destEtag.LastDocEtag;
if (currentLatestEtag != null && EtagUtil.IsGreaterThan(currentLatestEtag, lastDocEtag))
{
lastDocEtag = currentLatestEtag;
}
destEtag.LastDocEtag = lastDocEtag;
}
}
//We are done recording success for this batch so we can clear the reset dictionary
ResetRequested.Clear();
SaveNewReplicationStatus(localReplicationStatus);
}
finally
{
AfterReplicationCompleted(successes.Count);
RemoveUnusedPrefetchers(usedPrefetchers);
}
}
}
private void BackgroundSqlReplication()
{
int workCounter = 0;
while (Database.WorkContext.DoWork)
{
IsRunning = !shouldPause;
if (!IsRunning)
continue;
var config = GetConfiguredReplicationDestinations();
if (config.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var localReplicationStatus = GetReplicationStatus();
var relevantConfigs = config.Where(x =>
{
if (x.Disabled)
return false;
var sqlReplicationStatistics = statistics.GetOrDefault(x.Name);
if (sqlReplicationStatistics == null)
return true;
return SystemTime.UtcNow >= sqlReplicationStatistics.SuspendUntil;
}) // have error or the timeout expired
.ToList();
var configGroups = SqlReplicationClassifier.GroupConfigs(relevantConfigs, c => GetLastEtagFor(localReplicationStatus, c));
if (configGroups.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var usedPrefetchers = new ConcurrentSet<PrefetchingBehavior>();
var groupedConfigs = configGroups
.Select(x =>
{
var result = new SqlConfigGroup
{
LastReplicatedEtag = x.Key,
ConfigsToWorkOn = x.Value,
PrefetchingBehavior = GetPrefetcherFor(x.Key, usedPrefetchers)
};
result.PrefetchingBehavior.AdditionalInfo = string.Format("Default prefetcher: {0}. For sql config group: [Configs: {1}, LastReplicatedEtag: {2}]", result.PrefetchingBehavior == defaultPrefetchingBehavior, string.Join(", ", result.ConfigsToWorkOn.Select(y => y.Name)), result.LastReplicatedEtag);
return result;
})
.ToList();
var successes = new ConcurrentQueue<Tuple<SqlReplicationConfigWithLastReplicatedEtag, Etag>>();
var waitForWork = new bool[groupedConfigs.Count];
try
{
BackgroundTaskExecuter.Instance.ExecuteAll(Database.WorkContext, groupedConfigs, (sqlConfigGroup, i) =>
{
Database.WorkContext.CancellationToken.ThrowIfCancellationRequested();
var prefetchingBehavior = sqlConfigGroup.PrefetchingBehavior;
var configsToWorkOn = sqlConfigGroup.ConfigsToWorkOn;
List<JsonDocument> documents;
using (prefetchingBehavior.DocumentBatchFrom(sqlConfigGroup.LastReplicatedEtag, out documents))
{
Etag latestEtag = null, lastBatchEtag = null;
if (documents.Count != 0)
lastBatchEtag = documents[documents.Count - 1].Etag;
var replicationDuration = Stopwatch.StartNew();
documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here
if (documents.Count != 0)
latestEtag = documents[documents.Count - 1].Etag;
documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false);
var deletedDocsByConfig = new Dictionary<SqlReplicationConfig, List<ListItem>>();
foreach (var configToWorkOn in configsToWorkOn)
{
var cfg = configToWorkOn;
Database.TransactionalStorage.Batch(accessor =>
{
deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg),
cfg.LastReplicatedEtag,
latestEtag,
MaxNumberOfDeletionsToReplicate + 1)
.ToList();
});
}
// No documents AND there aren't any deletes to replicate
if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0)
{
// so we filtered some documents, let us update the etag about that.
if (latestEtag != null)
{
foreach (var configToWorkOn in configsToWorkOn)
successes.Enqueue(Tuple.Create(configToWorkOn, latestEtag));
}
else
{
waitForWork[i] = true;
}
return;
}
var itemsToReplicate = documents.Select(x =>
{
JsonDocument.EnsureIdInMetadata(x);
var doc = x.ToJson();
doc[Constants.DocumentIdFieldName] = x.Key;
return new ReplicatedDoc
{
Document = doc,
Etag = x.Etag,
Key = x.Key,
SerializedSizeOnDisk = x.SerializedSizeOnDisk
};
}).ToList();
try
{
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, configsToWorkOn, replicationConfig =>
{
try
{
var startTime = SystemTime.UtcNow;
var spRepTime = new Stopwatch();
spRepTime.Start();
var lastReplicatedEtag = replicationConfig.LastReplicatedEtag;
var deletedDocs = deletedDocsByConfig[replicationConfig];
var docsToReplicate = itemsToReplicate
.Where(x => lastReplicatedEtag.CompareTo(x.Etag) < 0) // haven't replicate the etag yet
.Where(document =>
{
var info = Database.Documents.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0)
{
Log.Debug(
"Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
document.Key, replicationConfig.Name, info.TouchedEtag);
return false;
}
}
return true;
});
if (deletedDocs.Count >= MaxNumberOfDeletionsToReplicate + 1)
docsToReplicate = docsToReplicate.Where(x => EtagUtil.IsGreaterThan(x.Etag, deletedDocs[deletedDocs.Count - 1].Etag) == false);
var docsToReplicateAsList = docsToReplicate.ToList();
var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicateAsList);
if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicateAsList.Count == 0)
currentLatestEtag = lastBatchEtag;
int countOfReplicatedItems = 0;
if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) &&
ReplicateChangesToDestination(replicationConfig, docsToReplicateAsList, out countOfReplicatedItems))
{
if (deletedDocs.Count > 0)
{
Database.TransactionalStorage.Batch(accessor =>
accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag));
}
successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag));
}
spRepTime.Stop();
var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick);
var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig);
sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems);
sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems);
sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds);
UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicateAsList.Count);
}
catch (Exception e)
{
Log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Exception = e.ToString(),
Title = "Sql Replication failure to replication",
Message = "Sql Replication could not replicate to " + replicationConfig.Name,
UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name
});
}
});
}
finally
{
prefetchingBehavior.CleanupDocuments(lastBatchEtag);
prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed);
}
}
});
if (successes.Count == 0)
{
if (waitForWork.All(x => x))
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
foreach (var t in successes)
{
var cfg = t.Item1;
var currentLatestEtag = t.Item2;
var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
if (destEtag == null)
{
localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
{
Name = cfg.Name,
LastDocEtag = currentLatestEtag ?? Etag.Empty
});
}
else
{
var lastDocEtag = destEtag.LastDocEtag;
if (currentLatestEtag != null && EtagUtil.IsGreaterThan(currentLatestEtag, lastDocEtag))
lastDocEtag = currentLatestEtag;
destEtag.LastDocEtag = lastDocEtag;
}
}
SaveNewReplicationStatus(localReplicationStatus);
}
finally
{
AfterReplicationCompleted(successes.Count);
RemoveUnusedPrefetchers(usedPrefetchers);
}
}
}
private void SetPrefetcherForIndexingGroup(SqlConfigGroup sqlConfig, ConcurrentSet<PrefetchingBehavior> usedPrefetchers)
{
sqlConfig.PrefetchingBehavior = TryGetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers) ??
TryGetDefaultPrefetcher(sqlConfig.LastReplicatedEtag, usedPrefetchers) ??
GetPrefetcherFor(sqlConfig.LastReplicatedEtag, usedPrefetchers);
sqlConfig.PrefetchingBehavior.AdditionalInfo =
string.Format("Default prefetcher: {0}. For sql config group: [Configs: {1}, LastReplicatedEtag: {2}]",
sqlConfig.PrefetchingBehavior == defaultPrefetchingBehavior,
string.Join(", ", sqlConfig.ConfigsToWorkOn.Select(y => y.Name)),
sqlConfig.LastReplicatedEtag);
}
