public void ShouldDisableCollectingDocsAfterCommit()
{
Etag last = Etag.Empty;
SystemTime.UtcDateTime = () => DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(15));
prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(Enumerable.Range(0, 5).Select(x =>
{
last = EtagUtil.Increment(last, 1);
return(new JsonDocument
{
Etag = last,
Key = x.ToString(CultureInfo.InvariantCulture)
});
}).ToArray());
last = EtagUtil.Increment(last, store.Configuration.MaxNumberOfItemsToProcessInSingleBatch);
prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(Enumerable.Range(0, 5).Select(x =>
{
last = EtagUtil.Increment(last, 1);
return(new JsonDocument
{
Etag = last,
Key = x.ToString(CultureInfo.InvariantCulture)
});
}).ToArray());
SystemTime.UtcDateTime = null;
var documentsBatchFrom = prefetchingBehavior.GetDocumentsBatchFrom(Etag.Empty);
prefetchingBehavior.CleanupDocuments(documentsBatchFrom.Last().Etag);
Assert.True(prefetchingBehavior.DisableCollectingDocumentsAfterCommit);
for (int i = 0; i < 5; i++)
{
last = EtagUtil.Increment(last, 1);
prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(new[]
{
new JsonDocument
{
Etag = last,
Key = i.ToString(CultureInfo.InvariantCulture)
},
});
}
Assert.Equal(0, prefetchingBehavior.InMemoryIndexingQueueSize);
}
public void CanMergeConsecutiveInMemoryUpdates()
{
var last = Guid.Empty;
for (int i = 0; i < 5; i++)
{
last = Etag.Increment(last, 1);
prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(new[]
{
new JsonDocument
{
Etag = last,
Key = i.ToString(CultureInfo.InvariantCulture)
},
});
}
Assert.Equal(5, prefetchingBehavior.GetDocumentsBatchFrom(Guid.Empty).Count);
}
private List <JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, JsonDocumentsToReplicate result)
{
var docsToReplicate = prefetchingBehavior.GetDocumentsBatchFrom(result.LastEtag);
Etag lastEtag = null;
if (docsToReplicate.Count > 0)
{
lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag;
}
return(docsToReplicate.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, lastEtag, 1024)
.Select(x => new JsonDocument
{
Etag = x.Etag,
Key = x.Key,
Metadata = x.Data,
DataAsJson = new RavenJObject()
}))
.OrderBy(x => x.Etag)
.ToList());
}
private void BackgroundSqlReplication()
{
int workCounter = 0;
while (Database.WorkContext.DoWork)
{
var config = GetConfiguredReplicationDestinations();
if (config.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var localReplicationStatus = GetReplicationStatus();
var relevantConfigs = config.Where(x =>
{
if (x.Disabled)
{
return(false);
}
var sqlReplicationStatistics = statistics.GetOrDefault(x.Name);
if (sqlReplicationStatistics == null)
{
return(true);
}
return(SystemTime.UtcNow >= sqlReplicationStatistics.LastErrorTime);
}) // have error or the timeout expired
.ToList();
if (relevantConfigs.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus);
if (leastReplicatedEtag == null)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var documents = prefetchingBehavior.GetDocumentsBatchFrom(leastReplicatedEtag);
Etag latestEtag = null, lastBatchEtag = null;
if (documents.Count != 0)
{
lastBatchEtag = documents[documents.Count - 1].Etag;
}
var replicationDuration = Stopwatch.StartNew();
documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here
if (documents.Count != 0)
{
latestEtag = documents[documents.Count - 1].Etag;
}
var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >();
foreach (var relevantConfig in relevantConfigs)
{
var cfg = relevantConfig;
Database.TransactionalStorage.Batch(accessor =>
{
deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg),
GetLastEtagFor(localReplicationStatus, cfg),
latestEtag,
1024)
.ToList();
});
}
// No documents AND there aren't any deletes to replicate
if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0)
{
if (latestEtag != null)
{
// so we filtered some documents, let us update the etag about that.
foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags)
{
if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0)
{
lastReplicatedEtag.LastDocEtag = latestEtag;
}
}
latestEtag = Etag.Max(latestEtag, lastBatchEtag);
SaveNewReplicationStatus(localReplicationStatus, latestEtag);
}
else // no point in waiting if we just saved a new doc
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
}
continue;
}
var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >();
try
{
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig =>
{
try
{
var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig);
var deletedDocs = deletedDocsByConfig[replicationConfig];
var docsToReplicate = documents
.Where(x => lastReplicatedEtag.CompareTo(x.Etag) <= 0) // haven't replicate the etag yet
.ToList();
var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicate);
if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) &&
ReplicateChangesToDesintation(replicationConfig, docsToReplicate))
{
if (deletedDocs.Count > 0)
{
Database.TransactionalStorage.Batch(accessor =>
accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag));
}
successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag));
}
}
catch (Exception e)
{
log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Exception = e.ToString(),
Title = "Sql Replication failure to replication",
Message = "Sql Replication could not replicate to " + replicationConfig.Name,
UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name
});
}
});
if (successes.Count == 0)
{
continue;
}
foreach (var t in successes)
{
var cfg = t.Item1;
var currentLatestEtag = t.Item2;
var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
if (destEtag == null)
{
localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
{
Name = cfg.Name,
LastDocEtag = currentLatestEtag ?? Etag.Empty
});
}
else
{
destEtag.LastDocEtag = currentLatestEtag = currentLatestEtag ?? destEtag.LastDocEtag;
}
latestEtag = Etag.Max(latestEtag, currentLatestEtag);
}
latestEtag = Etag.Max(latestEtag, lastBatchEtag);
SaveNewReplicationStatus(localReplicationStatus, latestEtag);
}
finally
{
AfterReplicationCompleted(successes.Count);
var lastMinReplicatedEtag = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray())).ToEtag();
prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag);
prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed);
}
}
}
private void BackgroundSqlReplication()
{
int workCounter = 0;
while (Database.WorkContext.DoWork)
{
var config = GetConfiguredReplicationDestinations();
if (config.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var localReplicationStatus = GetReplicationStatus();
var leastReplicatedEtag = GetLeastReplicatedEtag(config, localReplicationStatus);
if (leastReplicatedEtag == null)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var relevantConfigs = config.Where(x =>
{
if (x.Disabled)
{
return(false);
}
var sqlReplicationStatistics = statistics.GetOrDefault(x.Name);
if (sqlReplicationStatistics == null)
{
return(true);
}
return(SystemTime.UtcNow >= sqlReplicationStatistics.LastErrorTime);
}) // have error or the timeout expired
.ToList();
if (relevantConfigs.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var documents = prefetchingBehavior.GetDocumentsBatchFrom(leastReplicatedEtag.Value);
documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here
var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >();
Guid?latestEtag = null;
if (documents.Count != 0)
{
latestEtag = documents[documents.Count - 1].Etag;
}
foreach (var relevantConfig in relevantConfigs)
{
var cfg = relevantConfig;
Database.TransactionalStorage.Batch(accessor =>
{
deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg),
GetLastEtagFor(localReplicationStatus, cfg),
latestEtag,
1024)
.ToList();
});
}
// No documents AND there aren't any deletes to replicate
if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var successes = new ConcurrentQueue <SqlReplicationConfig>();
try
{
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig =>
{
try
{
var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig);
var deletedDocs = deletedDocsByConfig[replicationConfig];
var docsToReplicate = documents
.Where(x => ByteArrayComparer.Instance.Compare(lastReplicatedEtag, x.Etag) <= 0) // haven't replicate the etag yet
.ToList();
latestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicate);
if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) &&
ReplicateChangesToDesintation(replicationConfig, docsToReplicate))
{
if (deletedDocs.Count > 0)
{
Database.TransactionalStorage.Batch(accessor =>
accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag));
}
successes.Enqueue(replicationConfig);
}
}
catch (Exception e)
{
log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Exception = e.ToString(),
Title = "Sql Replication failure to replication",
Message = "Sql Replication could not replicate to " + replicationConfig.Name,
UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name
});
}
});
if (successes.Count == 0)
{
continue;
}
foreach (var cfg in successes)
{
var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
if (destEtag == null)
{
localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
{
Name = cfg.Name,
LastDocEtag = latestEtag ?? Guid.Empty
});
}
else
{
destEtag.LastDocEtag = latestEtag ?? destEtag.LastDocEtag;
}
}
var obj = RavenJObject.FromObject(localReplicationStatus);
Database.Put(RavenSqlreplicationStatus, null, obj, new RavenJObject(), null);
}
finally
{
AfterReplicationCompleted(successes.Count);
}
}
}
private void BackgroundSqlReplication()
{
int workCounter = 0;
while (Database.WorkContext.DoWork)
{
var config = GetConfiguredReplicationDestinations();
if (config.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var localReplicationStatus = GetReplicationStatus();
var leastReplicatedEtag = GetLeastReplicatedEtag(config, localReplicationStatus);
if (leastReplicatedEtag == null)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var documents = prefetchingBehavior.GetDocumentsBatchFrom(leastReplicatedEtag.Value);
if (documents.Count == 0 ||
documents.All(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase))) // ignore changes for system docs
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var latestEtag = documents.Last().Etag.Value;
var relevantConfigs =
config
.Where(x => ByteArrayComparer.Instance.Compare(GetLastEtagFor(localReplicationStatus, x), latestEtag) <= 0) // haven't replicate the etag yet
.Where(x => SystemTime.UtcNow >= lastError.GetOrDefault(x.Name)) // have error or the timeout expired
.ToList();
if (relevantConfigs.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
try
{
var successes = new ConcurrentQueue <SqlReplicationConfig>();
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig =>
{
try
{
if (ReplicateToDesintation(replicationConfig, documents))
{
successes.Enqueue(replicationConfig);
}
}
catch (Exception e)
{
log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
}
});
if (successes.Count == 0)
{
continue;
}
foreach (var cfg in successes)
{
var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
if (destEtag == null)
{
localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
{
Name = cfg.Name,
LastDocEtag = latestEtag
});
}
else
{
destEtag.LastDocEtag = latestEtag;
}
}
var obj = RavenJObject.FromObject(localReplicationStatus);
Database.Put(RavenSqlreplicationStatus, null, obj, new RavenJObject(), null);
}
finally
{
AfterReplicationCompleted();
}
}
}
