public void ShouldDisableCollectingDocsAfterCommit() { Etag last = Etag.Empty; SystemTime.UtcDateTime = () => DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(15)); prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(Enumerable.Range(0, 5).Select(x => { last = EtagUtil.Increment(last, 1); return(new JsonDocument { Etag = last, Key = x.ToString(CultureInfo.InvariantCulture) }); }).ToArray()); last = EtagUtil.Increment(last, store.Configuration.MaxNumberOfItemsToProcessInSingleBatch); prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(Enumerable.Range(0, 5).Select(x => { last = EtagUtil.Increment(last, 1); return(new JsonDocument { Etag = last, Key = x.ToString(CultureInfo.InvariantCulture) }); }).ToArray()); SystemTime.UtcDateTime = null; var documentsBatchFrom = prefetchingBehavior.GetDocumentsBatchFrom(Etag.Empty); prefetchingBehavior.CleanupDocuments(documentsBatchFrom.Last().Etag); Assert.True(prefetchingBehavior.DisableCollectingDocumentsAfterCommit); for (int i = 0; i < 5; i++) { last = EtagUtil.Increment(last, 1); prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(new[] { new JsonDocument { Etag = last, Key = i.ToString(CultureInfo.InvariantCulture) }, }); } Assert.Equal(0, prefetchingBehavior.InMemoryIndexingQueueSize); }
public void CanMergeConsecutiveInMemoryUpdates() { var last = Guid.Empty; for (int i = 0; i < 5; i++) { last = Etag.Increment(last, 1); prefetchingBehavior.AfterStorageCommitBeforeWorkNotifications(new[] { new JsonDocument { Etag = last, Key = i.ToString(CultureInfo.InvariantCulture) }, }); } Assert.Equal(5, prefetchingBehavior.GetDocumentsBatchFrom(Guid.Empty).Count); }
private List <JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, JsonDocumentsToReplicate result) { var docsToReplicate = prefetchingBehavior.GetDocumentsBatchFrom(result.LastEtag); Etag lastEtag = null; if (docsToReplicate.Count > 0) { lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag; } return(docsToReplicate.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, lastEtag, 1024) .Select(x => new JsonDocument { Etag = x.Etag, Key = x.Key, Metadata = x.Data, DataAsJson = new RavenJObject() })) .OrderBy(x => x.Etag) .ToList()); }
private void BackgroundSqlReplication() { int workCounter = 0; while (Database.WorkContext.DoWork) { var config = GetConfiguredReplicationDestinations(); if (config.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var localReplicationStatus = GetReplicationStatus(); var relevantConfigs = config.Where(x => { if (x.Disabled) { return(false); } var sqlReplicationStatistics = statistics.GetOrDefault(x.Name); if (sqlReplicationStatistics == null) { return(true); } return(SystemTime.UtcNow >= sqlReplicationStatistics.LastErrorTime); }) // have error or the timeout expired .ToList(); if (relevantConfigs.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus); if (leastReplicatedEtag == null) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var documents = prefetchingBehavior.GetDocumentsBatchFrom(leastReplicatedEtag); Etag latestEtag = null, lastBatchEtag = null; if (documents.Count != 0) { lastBatchEtag = documents[documents.Count - 1].Etag; } var replicationDuration = Stopwatch.StartNew(); documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here if (documents.Count != 0) { latestEtag = documents[documents.Count - 1].Etag; } var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >(); foreach (var relevantConfig in relevantConfigs) { var cfg = relevantConfig; Database.TransactionalStorage.Batch(accessor => { deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg), GetLastEtagFor(localReplicationStatus, cfg), latestEtag, 1024) .ToList(); }); } // No documents AND there aren't any deletes to replicate if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0) { if (latestEtag != null) { // so we filtered some documents, let us update the etag about that. foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags) { if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0) { lastReplicatedEtag.LastDocEtag = latestEtag; } } latestEtag = Etag.Max(latestEtag, lastBatchEtag); SaveNewReplicationStatus(localReplicationStatus, latestEtag); } else // no point in waiting if we just saved a new doc { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); } continue; } var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >(); try { BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig => { try { var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig); var deletedDocs = deletedDocsByConfig[replicationConfig]; var docsToReplicate = documents .Where(x => lastReplicatedEtag.CompareTo(x.Etag) <= 0) // haven't replicate the etag yet .ToList(); var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicate); if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) && ReplicateChangesToDesintation(replicationConfig, docsToReplicate)) { if (deletedDocs.Count > 0) { Database.TransactionalStorage.Batch(accessor => accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag)); } successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag)); } } catch (Exception e) { log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Sql Replication failure to replication", Message = "Sql Replication could not replicate to " + replicationConfig.Name, UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name }); } }); if (successes.Count == 0) { continue; } foreach (var t in successes) { var cfg = t.Item1; var currentLatestEtag = t.Item2; var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase)); if (destEtag == null) { localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag { Name = cfg.Name, LastDocEtag = currentLatestEtag ?? Etag.Empty }); } else { destEtag.LastDocEtag = currentLatestEtag = currentLatestEtag ?? destEtag.LastDocEtag; } latestEtag = Etag.Max(latestEtag, currentLatestEtag); } latestEtag = Etag.Max(latestEtag, lastBatchEtag); SaveNewReplicationStatus(localReplicationStatus, latestEtag); } finally { AfterReplicationCompleted(successes.Count); var lastMinReplicatedEtag = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray())).ToEtag(); prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag); prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed); } } }
private void BackgroundSqlReplication() { int workCounter = 0; while (Database.WorkContext.DoWork) { var config = GetConfiguredReplicationDestinations(); if (config.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var localReplicationStatus = GetReplicationStatus(); var leastReplicatedEtag = GetLeastReplicatedEtag(config, localReplicationStatus); if (leastReplicatedEtag == null) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var relevantConfigs = config.Where(x => { if (x.Disabled) { return(false); } var sqlReplicationStatistics = statistics.GetOrDefault(x.Name); if (sqlReplicationStatistics == null) { return(true); } return(SystemTime.UtcNow >= sqlReplicationStatistics.LastErrorTime); }) // have error or the timeout expired .ToList(); if (relevantConfigs.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var documents = prefetchingBehavior.GetDocumentsBatchFrom(leastReplicatedEtag.Value); documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >(); Guid?latestEtag = null; if (documents.Count != 0) { latestEtag = documents[documents.Count - 1].Etag; } foreach (var relevantConfig in relevantConfigs) { var cfg = relevantConfig; Database.TransactionalStorage.Batch(accessor => { deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg), GetLastEtagFor(localReplicationStatus, cfg), latestEtag, 1024) .ToList(); }); } // No documents AND there aren't any deletes to replicate if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var successes = new ConcurrentQueue <SqlReplicationConfig>(); try { BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig => { try { var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig); var deletedDocs = deletedDocsByConfig[replicationConfig]; var docsToReplicate = documents .Where(x => ByteArrayComparer.Instance.Compare(lastReplicatedEtag, x.Etag) <= 0) // haven't replicate the etag yet .ToList(); latestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicate); if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) && ReplicateChangesToDesintation(replicationConfig, docsToReplicate)) { if (deletedDocs.Count > 0) { Database.TransactionalStorage.Batch(accessor => accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag)); } successes.Enqueue(replicationConfig); } } catch (Exception e) { log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Sql Replication failure to replication", Message = "Sql Replication could not replicate to " + replicationConfig.Name, UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name }); } }); if (successes.Count == 0) { continue; } foreach (var cfg in successes) { var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase)); if (destEtag == null) { localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag { Name = cfg.Name, LastDocEtag = latestEtag ?? Guid.Empty }); } else { destEtag.LastDocEtag = latestEtag ?? destEtag.LastDocEtag; } } var obj = RavenJObject.FromObject(localReplicationStatus); Database.Put(RavenSqlreplicationStatus, null, obj, new RavenJObject(), null); } finally { AfterReplicationCompleted(successes.Count); } } }
private void BackgroundSqlReplication() { int workCounter = 0; while (Database.WorkContext.DoWork) { var config = GetConfiguredReplicationDestinations(); if (config.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var localReplicationStatus = GetReplicationStatus(); var leastReplicatedEtag = GetLeastReplicatedEtag(config, localReplicationStatus); if (leastReplicatedEtag == null) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var documents = prefetchingBehavior.GetDocumentsBatchFrom(leastReplicatedEtag.Value); if (documents.Count == 0 || documents.All(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase))) // ignore changes for system docs { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var latestEtag = documents.Last().Etag.Value; var relevantConfigs = config .Where(x => ByteArrayComparer.Instance.Compare(GetLastEtagFor(localReplicationStatus, x), latestEtag) <= 0) // haven't replicate the etag yet .Where(x => SystemTime.UtcNow >= lastError.GetOrDefault(x.Name)) // have error or the timeout expired .ToList(); if (relevantConfigs.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } try { var successes = new ConcurrentQueue <SqlReplicationConfig>(); BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig => { try { if (ReplicateToDesintation(replicationConfig, documents)) { successes.Enqueue(replicationConfig); } } catch (Exception e) { log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e); } }); if (successes.Count == 0) { continue; } foreach (var cfg in successes) { var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase)); if (destEtag == null) { localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag { Name = cfg.Name, LastDocEtag = latestEtag }); } else { destEtag.LastDocEtag = latestEtag; } } var obj = RavenJObject.FromObject(localReplicationStatus); Database.Put(RavenSqlreplicationStatus, null, obj, new RavenJObject(), null); } finally { AfterReplicationCompleted(); } } }