public void GetDocumentsBatchFromShouldReturnUniqueResults2() { var prefetcher = CreatePrefetcher(); AddDocumentsToTransactionalStorage(prefetcher.TransactionalStorage, 100); var documents = prefetcher.PrefetchingBehavior.GetDocumentsBatchFrom(Etag.Empty, 1); var document = documents.Single(); AddDocumentResult result = null; prefetcher.TransactionalStorage.Batch(accessor => result = accessor.Documents.AddDocument("keys/2", null, document.DataAsJson, document.Metadata)); documents = prefetcher.PrefetchingBehavior.GetDocumentsBatchFrom(document.Etag, 200); Assert.Equal(99, documents.Count); var prevEtag = Etag.Empty; var keys = new HashSet <string>(StringComparer.OrdinalIgnoreCase); foreach (var doc in documents) { Assert.True(EtagUtil.IsGreaterThan(document.Etag, prevEtag)); Assert.True(keys.Add(doc.Key)); } }
public Etag GetBestNextDocumentEtag(Etag etag) { if (etag == null) { throw new ArgumentNullException("etag"); } using (var iter = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag) .Iterate(Snapshot, writeBatch.Value)) { if (!iter.Seek((Slice)etag.ToString()) && !iter.Seek(Slice.BeforeAllKeys)) //if parameter etag not found, scan from beginning. if empty --> return original etag { return(etag); } do { var docEtag = Etag.Parse(iter.CurrentKey.ToString()); if (EtagUtil.IsGreaterThan(docEtag, etag)) { return(docEtag); } } while (iter.MoveNext()); } return(etag); //if not found, return the original etag }
private bool IsIndexStateValid(LuceneDirectory luceneDirectory) { // 1. If commitData is null it means that there were no commits, so just in case we are resetting to Etag.Empty // 2. If no 'LastEtag' in commitData then we consider it an invalid index // 3. If 'LastEtag' is present (and valid), then resetting to it (if it is lower than lastStoredEtag) var commitData = IndexReader.GetCommitUserData(luceneDirectory); string value; Etag lastEtag = null; if (commitData != null && commitData.TryGetValue("LastEtag", out value)) { Etag.TryParse(value, out lastEtag); // etag will be null if parsing will fail } var lastStoredEtag = GetLastEtagForIndex() ?? Etag.Empty; lastEtag = lastEtag ?? Etag.Empty; if (EtagUtil.IsGreaterThan(lastEtag, lastStoredEtag)) { return(false); } var checkIndex = new CheckIndex(luceneDirectory); var status = checkIndex.CheckIndex_Renamed_Method(); return(status.clean); }
public void AbstractScriptedIndexCreationTaskWillResetIndexIfDocumentHasChanged() { using (var store = NewDocumentStore()) { using (var session = store.OpenSession()) { session.Store(new Person { Name = "Name1" }); session.SaveChanges(); } new People_By_Name().Execute(store); var index = new People_By_Name_With_Scripts(); store.DatabaseCommands.Put(ScriptedIndexResults.IdPrefix + index.IndexName, null, new RavenJObject(), new RavenJObject()); WaitForIndexing(store); var stats = store.DatabaseCommands.GetStatistics(); var indexStats = stats.Indexes.First(x => x.Name == index.IndexName); Assert.True(EtagUtil.IsGreaterThan(indexStats.LastIndexedEtag, Etag.Empty)); store.DatabaseCommands.Admin.StopIndexing(); index.Execute(store); stats = store.DatabaseCommands.GetStatistics(); indexStats = stats.Indexes.First(x => x.Name == index.IndexName); Assert.True(indexStats.LastIndexedEtag.Equals(Etag.Empty)); } }
public void AbstractScriptedIndexCreationTaskWillNotResetIndexIfNothingHasChanged() { using (var store = NewDocumentStore()) { using (var session = store.OpenSession()) { session.Store(new Person { Name = "Name1" }); session.SaveChanges(); } var index = new People_By_Name_With_Scripts(); index.Execute(store); WaitForIndexing(store); var stats = store.DatabaseCommands.GetStatistics(); var indexStats = stats.Indexes.First(x => x.Name == index.IndexName); var lastIndexedEtag = indexStats.LastIndexedEtag; Assert.True(EtagUtil.IsGreaterThan(lastIndexedEtag, Etag.Empty)); store.DatabaseCommands.Admin.StopIndexing(); index.Execute(store); stats = store.DatabaseCommands.GetStatistics(); indexStats = stats.Indexes.First(x => x.Name == index.IndexName); Assert.True(indexStats.LastIndexedEtag.Equals(lastIndexedEtag) || EtagUtil.IsGreaterThan(indexStats.LastIndexedEtag, lastIndexedEtag)); } }
private void SaveSynchronizationSourceInformation(FileSystemInfo sourceFileSystem, Etag lastSourceEtag) { var lastSynchronizationInformation = GetLastSynchronization(sourceFileSystem.Id); if (EtagUtil.IsGreaterThan(lastSynchronizationInformation.LastSourceFileEtag, lastSourceEtag)) { return; } var synchronizationSourceInfo = new SourceSynchronizationInformation { LastSourceFileEtag = lastSourceEtag, SourceServerUrl = sourceFileSystem.Url, DestinationServerId = Storage.Id }; var key = SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + sourceFileSystem.Id; Storage.Batch(accessor => accessor.SetConfig(key, JsonExtensions.ToJObject(synchronizationSourceInfo))); if (Log.IsDebugEnabled) { Log.Debug("Saved last synchronized file ETag {0} from {1} ({2})", lastSourceEtag, sourceFileSystem.Url, sourceFileSystem.Id); } }
public void GetDocumentsBatchFromShouldReturnResultsOrderedByEtag() { var prefetcher = CreatePrefetcher(); AddDocumentsToTransactionalStorage(prefetcher.TransactionalStorage, 100); var documents = prefetcher.PrefetchingBehavior.GetDocumentsBatchFrom(Etag.Empty, 100); var prevEtag = Etag.Empty; foreach (var document in documents) { Assert.True(EtagUtil.IsGreaterThan(document.Etag, prevEtag)); prevEtag = document.Etag; } }
public IEnumerable <JsonDocument> GetDocumentsAfter(Etag etag, int take, CancellationToken cancellationToken, long?maxSize = null, Etag untilEtag = null, TimeSpan?timeout = null) { Api.JetSetCurrentIndex(session, Documents, "by_etag"); Api.MakeKey(session, Documents, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey); if (Api.TrySeek(session, Documents, SeekGrbit.SeekGT) == false) { yield break; } long totalSize = 0; int count = 0; Stopwatch duration = null; if (timeout != null) { duration = Stopwatch.StartNew(); } do { cancellationToken.ThrowIfCancellationRequested(); if (untilEtag != null && count > 0) { var docEtag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"])); if (EtagUtil.IsGreaterThan(docEtag, untilEtag)) { yield break; } } var readCurrentDocument = ReadCurrentDocument(); totalSize += readCurrentDocument.SerializedSizeOnDisk; if (maxSize != null && totalSize > maxSize.Value) { yield return(readCurrentDocument); yield break; } yield return(readCurrentDocument); count++; if (timeout != null) { if (duration.Elapsed > timeout.Value) { yield break; } } } while (Api.TryMoveNext(session, Documents) && count < take); }
public IEnumerable <JsonDocument> GetDocumentsAfterWithIdStartingWith(Etag etag, string idPrefix, int take, CancellationToken cancellationToken, long?maxSize = null, Etag untilEtag = null, TimeSpan?timeout = null, Action <Etag> lastProcessedDocument = null, Reference <bool> earlyExit = null) { if (earlyExit != null) { earlyExit.Value = false; } if (take < 0) { throw new ArgumentException("must have zero or positive value", "take"); } if (take == 0) { yield break; } if (string.IsNullOrEmpty(etag)) { throw new ArgumentNullException("etag"); } Stopwatch duration = null; if (timeout != null) { duration = Stopwatch.StartNew(); } Etag lastDocEtag = null; using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag) .Iterate(Snapshot, writeBatch.Value)) { var slice = (Slice)etag.ToString(); if (iterator.Seek(slice) == false) { yield break; } if (iterator.CurrentKey.Equals(slice)) // need gt, not ge { if (iterator.MoveNext() == false) { yield break; } } long fetchedDocumentTotalSize = 0; int fetchedDocumentCount = 0; Etag docEtag = etag; do { cancellationToken.ThrowIfCancellationRequested(); docEtag = Etag.Parse(iterator.CurrentKey.ToString()); // We can skip many documents so the timeout should be at the start of the process to be executed. if (timeout != null) { if (duration.Elapsed > timeout.Value) { if (earlyExit != null) { earlyExit.Value = true; } break; } } if (untilEtag != null) { // This is not a failure, we are just ahead of when we expected to. if (EtagUtil.IsGreaterThan(docEtag, untilEtag)) { break; } } var key = GetKeyFromCurrent(iterator); if (!string.IsNullOrEmpty(idPrefix)) { if (!key.StartsWith(idPrefix, StringComparison.OrdinalIgnoreCase)) { // We assume that we have processed it because it is not of our interest. lastDocEtag = docEtag; continue; } } var document = DocumentByKey(key); if (document == null) //precaution - should never be true { if (SkipConsistencyCheck) { continue; } throw new InvalidDataException(string.Format("Data corruption - the key = '{0}' was found in the documents index, but matching document was not found", key)); } if (!document.Etag.Equals(docEtag) && !SkipConsistencyCheck) { throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between document and its index", key)); } fetchedDocumentTotalSize += document.SerializedSizeOnDisk; fetchedDocumentCount++; yield return(document); lastDocEtag = docEtag; if (maxSize.HasValue && fetchedDocumentTotalSize >= maxSize) { if (untilEtag != null && earlyExit != null) { earlyExit.Value = true; } break; } if (fetchedDocumentCount >= take) { if (untilEtag != null && earlyExit != null) { earlyExit.Value = true; } break; } } while (iterator.MoveNext()); } // We notify the last that we considered. if (lastProcessedDocument != null) { lastProcessedDocument(lastDocEtag); } }
public HttpResponseMessage ExplainGet(string docId) { if (string.IsNullOrEmpty(docId)) { return(GetMessageWithString("Document key is required.", HttpStatusCode.BadRequest)); } var destinationUrl = GetQueryStringValue("destinationUrl"); if (string.IsNullOrEmpty(destinationUrl)) { return(GetMessageWithString("Destination url is required.", HttpStatusCode.BadRequest)); } var databaseName = GetQueryStringValue("databaseName"); if (string.IsNullOrEmpty(databaseName)) { return(GetMessageWithString("Destination database name is required.", HttpStatusCode.BadRequest)); } var result = new ReplicationExplanationForDocument { Key = docId, Destination = new ReplicationExplanationForDocument.DestinationInformation { Url = destinationUrl, DatabaseName = databaseName } }; var destinations = ReplicationTask.GetReplicationDestinations(x => string.Equals(x.Url, destinationUrl, StringComparison.OrdinalIgnoreCase) && string.Equals(x.Database, databaseName, StringComparison.OrdinalIgnoreCase)); if (destinations == null || destinations.Length == 0) { result.Message = string.Format("Could not find replication destination for a given url ('{0}') and database ('{1}').", destinationUrl, databaseName); return(GetMessageWithObject(result)); } if (destinations.Length > 1) { result.Message = string.Format("There is more than one replication destination for a given url ('{0}') and database ('{1}').", destinationUrl, databaseName); return(GetMessageWithObject(result)); } var destination = destinations[0]; var destinationsReplicationInformationForSource = ReplicationTask.GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) { result.Message = "Could not connect to destination server."; return(GetMessageWithObject(result)); } var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); result.Destination.ServerInstanceId = destinationId; result.Destination.LastDocumentEtag = destinationsReplicationInformationForSource.LastDocumentEtag; var document = Database.Documents.Get(docId, null); if (document == null) { result.Message = string.Format("Document with given key ('{0}') does not exist.", docId); return(GetMessageWithObject(result)); } result.Key = document.Key; result.Etag = document.Etag; string reason; if (destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) == false) { result.Message = reason; return(GetMessageWithObject(result)); } reason = EtagUtil.IsGreaterThan(document.Etag, destinationsReplicationInformationForSource.LastDocumentEtag) ? "Document will be replicated." : "Document should have been replicated."; result.Message = reason; return(GetMessageWithObject(result)); }
public IEnumerable <JsonDocument> GetDocumentsAfterWithIdStartingWith(Etag etag, string idPrefix, int take, CancellationToken cancellationToken, long?maxSize = null, Etag untilEtag = null, TimeSpan?timeout = null) { Api.JetSetCurrentIndex(session, Documents, "by_etag"); Api.MakeKey(session, Documents, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey); if (Api.TrySeek(session, Documents, SeekGrbit.SeekGT) == false) { yield break; } long totalSize = 0; int count = 0; Stopwatch duration = null; if (timeout != null) { duration = Stopwatch.StartNew(); } do { cancellationToken.ThrowIfCancellationRequested(); // We can skip many documents so the timeout should be at the start of the process to be executed. if (timeout != null) { if (duration.Elapsed > timeout.Value) { yield break; } } if (untilEtag != null && count > 0) { var docEtag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"])); if (EtagUtil.IsGreaterThan(docEtag, untilEtag)) { yield break; } } var key = Api.RetrieveColumnAsString(session, Documents, tableColumnsCache.DocumentsColumns["key"], Encoding.Unicode); if (!string.IsNullOrEmpty(idPrefix)) { // Check if the ignore case is correct. if (!key.StartsWith(idPrefix, StringComparison.OrdinalIgnoreCase)) { continue; } } var readCurrentDocument = ReadCurrentDocument(key); totalSize += readCurrentDocument.SerializedSizeOnDisk; yield return(readCurrentDocument); if (maxSize != null && totalSize > maxSize.Value) { yield break; } count++; } while (Api.TryMoveNext(session, Documents) && count < take); }
public IEnumerable <AttachmentInformation> GetAttachmentsAfter(Etag value, int take, long maxTotalSize) { if (take < 0) { throw new ArgumentException("must have zero or positive value", "take"); } if (maxTotalSize < 0) { throw new ArgumentException("must have zero or positive value", "maxTotalSize"); } if (take == 0) { yield break; //edge case } using (var iter = attachmentsTable.GetIndex(Tables.Attachments.Indices.ByEtag) .Iterate(Snapshot, writeBatch.Value)) { if (!iter.Seek(Slice.BeforeAllKeys)) { yield break; } var fetchedDocumentCount = 0; var fetchedTotalSize = 0; do { if (iter.CurrentKey == null || iter.CurrentKey.Equals(Slice.Empty)) { yield break; } var attachmentEtag = Etag.Parse(iter.CurrentKey.ToString()); if (!EtagUtil.IsGreaterThan(attachmentEtag, value)) { continue; } string key; using (var keyStream = iter.CreateReaderForCurrent().AsStream()) key = keyStream.ReadStringWithoutPrefix(); var attachmentInfo = AttachmentInfoByKey(key); if (!attachmentInfo.Etag.Equals(attachmentEtag)) { throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between attachment and its indice", key)); } fetchedTotalSize += attachmentInfo.Size; fetchedDocumentCount++; if (fetchedTotalSize > maxTotalSize) { yield break; } if (fetchedDocumentCount >= take || fetchedTotalSize == maxTotalSize) { yield return(attachmentInfo); yield break; } yield return(attachmentInfo); } while (iter.MoveNext()); } }
private void BackgroundSqlReplication() { int workCounter = 0; while (Database.WorkContext.DoWork) { IsRunning = !shouldPause; if (!IsRunning) { continue; } var config = GetConfiguredReplicationDestinations(); if (config.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var localReplicationStatus = GetReplicationStatus(); var relevantConfigs = config.Where(x => { if (x.Disabled) { return(false); } var sqlReplicationStatistics = statistics.GetOrDefault(x.Name); if (sqlReplicationStatistics == null) { return(true); } return(SystemTime.UtcNow >= sqlReplicationStatistics.SuspendUntil); }) // have error or the timeout expired .ToList(); if (relevantConfigs.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus); if (leastReplicatedEtag == null) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } List <JsonDocument> documents; using (prefetchingBehavior.DocumentBatchFrom(leastReplicatedEtag, out documents)) { Etag latestEtag = null, lastBatchEtag = null; if (documents.Count != 0) { lastBatchEtag = documents[documents.Count - 1].Etag; } var replicationDuration = Stopwatch.StartNew(); documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here if (documents.Count != 0) { latestEtag = documents[documents.Count - 1].Etag; } documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false); var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >(); foreach (var relevantConfig in relevantConfigs) { var cfg = relevantConfig; Database.TransactionalStorage.Batch(accessor => { deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg), GetLastEtagFor(localReplicationStatus, cfg), latestEtag, MaxNumberOfDeletionsToReplicate + 1) .ToList(); }); } // No documents AND there aren't any deletes to replicate if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0) { if (latestEtag != null) { // so we filtered some documents, let us update the etag about that. foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags) { if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0) { lastReplicatedEtag.LastDocEtag = latestEtag; } } latestEtag = Etag.Max(latestEtag, lastBatchEtag); SaveNewReplicationStatus(localReplicationStatus); } else // no point in waiting if we just saved a new doc { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); } continue; } var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >(); try { var itemsToReplicate = documents.Select(x => { JsonDocument.EnsureIdInMetadata(x); var doc = x.ToJson(); doc[Constants.DocumentIdFieldName] = x.Key; return(new ReplicatedDoc { Document = doc, Etag = x.Etag, Key = x.Key, SerializedSizeOnDisk = x.SerializedSizeOnDisk }); }).ToList(); BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig => { try { var startTime = SystemTime.UtcNow; Stopwatch spRepTime = new Stopwatch(); spRepTime.Start(); var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig); var deletedDocs = deletedDocsByConfig[replicationConfig]; var docsToReplicate = itemsToReplicate .Where(x => lastReplicatedEtag.CompareTo(x.Etag) < 0) // haven't replicate the etag yet .Where(document => { var info = Database.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0) { log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, replicationConfig.Name, info.TouchedEtag); return(false); } } return(true); }); if (deletedDocs.Count >= MaxNumberOfDeletionsToReplicate + 1) { docsToReplicate = docsToReplicate.Where(x => EtagUtil.IsGreaterThan(x.Etag, deletedDocs[deletedDocs.Count - 1].Etag) == false); } var docsToReplicateAsList = docsToReplicate.ToList(); var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicateAsList); if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicateAsList.Count == 0) { currentLatestEtag = lastBatchEtag; } int countOfReplicatedItems = 0; if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) && ReplicateChangesToDestination(replicationConfig, docsToReplicateAsList, out countOfReplicatedItems)) { if (deletedDocs.Count > 0) { Database.TransactionalStorage.Batch(accessor => accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag)); } successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag)); } spRepTime.Stop(); var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick); var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig); sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds); UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicateAsList.Count); } catch (Exception e) { log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Sql Replication failure to replication", Message = "Sql Replication could not replicate to " + replicationConfig.Name, UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name }); } }); if (successes.Count == 0) { continue; } foreach (var t in successes) { var cfg = t.Item1; var currentLatestEtag = t.Item2; var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase)); if (destEtag == null) { localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag { Name = cfg.Name, LastDocEtag = currentLatestEtag ?? Etag.Empty }); } else { var lastDocEtag = destEtag.LastDocEtag; if (currentLatestEtag != null && EtagUtil.IsGreaterThan(currentLatestEtag, lastDocEtag)) { lastDocEtag = currentLatestEtag; } destEtag.LastDocEtag = lastDocEtag; } } SaveNewReplicationStatus(localReplicationStatus); } finally { AfterReplicationCompleted(successes.Count); var min = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray())); if (min != null) { var lastMinReplicatedEtag = min.ToEtag(); prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag); prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed); } } } } }
public IEnumerable <JsonDocument> GetDocumentsAfterWithIdStartingWith(Etag etag, string idPrefix, int take, CancellationToken cancellationToken, long?maxSize = null, Etag untilEtag = null, TimeSpan?timeout = null, Action <Etag> lastProcessedDocument = null, Reference <bool> earlyExit = null) { if (earlyExit != null) { earlyExit.Value = false; } Api.JetSetCurrentIndex(session, Documents, "by_etag"); Api.MakeKey(session, Documents, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey); if (Api.TrySeek(session, Documents, SeekGrbit.SeekGT) == false) { yield break; } long totalSize = 0; int fetchedDocumentCount = 0; Stopwatch duration = null; if (timeout != null) { duration = Stopwatch.StartNew(); } Etag lastDocEtag = null; Etag docEtag = etag; do { cancellationToken.ThrowIfCancellationRequested(); docEtag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"])); // We can skip many documents so the timeout should be at the start of the process to be executed. if (timeout != null) { if (duration.Elapsed > timeout.Value) { if (earlyExit != null) { earlyExit.Value = true; } break; } } if (untilEtag != null && fetchedDocumentCount > 0) { // This is not a failure, we are just ahead of when we expected to. if (EtagUtil.IsGreaterThan(docEtag, untilEtag)) { break; } } var key = Api.RetrieveColumnAsString(session, Documents, tableColumnsCache.DocumentsColumns["key"], Encoding.Unicode); if (!string.IsNullOrEmpty(idPrefix)) { if (!key.StartsWith(idPrefix, StringComparison.OrdinalIgnoreCase)) { // We assume that we have processed it because it is not of our interest. lastDocEtag = docEtag; continue; } } var readCurrentDocument = ReadCurrentDocument(key); totalSize += readCurrentDocument.SerializedSizeOnDisk; fetchedDocumentCount++; yield return(readCurrentDocument); lastDocEtag = docEtag; if (maxSize != null && totalSize > maxSize.Value) { if (untilEtag != null && earlyExit != null) { earlyExit.Value = true; } break; } if (fetchedDocumentCount >= take) { if (untilEtag != null && earlyExit != null) { earlyExit.Value = true; } break; } }while (Api.TryMoveNext(session, Documents)); // We notify the last that we considered. if (lastProcessedDocument != null) { lastProcessedDocument(lastDocEtag); } }
private void StreamToClient(long id, SubscriptionActions subscriptions, Stream stream) { var sentDocuments = false; var bufferStream = new BufferedStream(stream, 1024 * 64); using (var writer = new JsonTextWriter(new StreamWriter(bufferStream))) { var options = subscriptions.GetBatchOptions(id); writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); using (var cts = new CancellationTokenSource()) using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) { Etag lastProcessedDocEtag = null; var batchSize = 0; var batchDocCount = 0; var processedDocuments = 0; var hasMoreDocs = false; var config = subscriptions.GetSubscriptionConfig(id); var startEtag = config.AckEtag; var criteria = config.Criteria; bool isPrefixCriteria = !string.IsNullOrWhiteSpace(criteria.KeyStartsWith); Func <JsonDocument, bool> addDocument = doc => { timeout.Delay(); if (doc == null) { // we only have this heartbeat when the streaming has gone on for a long time // and we haven't send anything to the user in a while (because of filtering, skipping, etc). writer.WriteRaw(Environment.NewLine); writer.Flush(); return(true); } processedDocuments++; // We cant continue because we have already maxed out the batch bytes size. if (options.MaxSize.HasValue && batchSize >= options.MaxSize) { return(false); } // We cant continue because we have already maxed out the amount of documents to send. if (batchDocCount >= options.MaxDocCount) { return(false); } // We can continue because we are ignoring system documents. if (doc.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)) { return(true); } // We can continue because we are ignoring the document as it doesn't fit the criteria. if (MatchCriteria(criteria, doc) == false) { return(true); } doc.ToJson().WriteTo(writer); writer.WriteRaw(Environment.NewLine); batchSize += doc.SerializedSizeOnDisk; batchDocCount++; return(true); // We get the next document }; int retries = 0; do { int lastIndex = processedDocuments; Database.TransactionalStorage.Batch(accessor => { // we may be sending a LOT of documents to the user, and most // of them aren't going to be relevant for other ops, so we are going to skip // the cache for that, to avoid filling it up very quickly using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache()) { if (isPrefixCriteria) { // If we don't get any document from GetDocumentsWithIdStartingWith it could be that we are in presence of a lagoon of uninteresting documents, so we are hitting a timeout. lastProcessedDocEtag = Database.Documents.GetDocumentsWithIdStartingWith(criteria.KeyStartsWith, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument); hasMoreDocs = false; } else { // It doesn't matter if we match the criteria or not, the document has been already processed. lastProcessedDocEtag = Database.Documents.GetDocuments(-1, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument); // If we don't get any document from GetDocuments it may be a signal that something is wrong. if (lastProcessedDocEtag == null) { hasMoreDocs = false; } else { var lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag(); hasMoreDocs = EtagUtil.IsGreaterThan(lastDocEtag, lastProcessedDocEtag); startEtag = lastProcessedDocEtag; } retries = lastIndex == batchDocCount ? retries : 0; } } }); if (lastIndex == processedDocuments) { if (retries == 3) { log.Warn("Subscription processing did not end up replicating any documents for 3 times in a row, stopping operation", retries); } else { log.Warn("Subscription processing did not end up replicating any documents, due to possible storage error, retry number: {0}", retries); } retries++; } }while (retries < 3 && hasMoreDocs && batchDocCount < options.MaxDocCount && (options.MaxSize.HasValue == false || batchSize < options.MaxSize)); writer.WriteEndArray(); if (batchDocCount > 0 || isPrefixCriteria) { writer.WritePropertyName("LastProcessedEtag"); writer.WriteValue(lastProcessedDocEtag.ToString()); sentDocuments = true; } writer.WriteEndObject(); writer.Flush(); bufferStream.Flush(); } } if (sentDocuments) { subscriptions.UpdateBatchSentTime(id); } }
private void StreamToClient(long id, SubscriptionActions subscriptions, Stream stream) { var sentDocuments = false; using (var streamWriter = new StreamWriter(stream)) using (var writer = new JsonTextWriter(streamWriter)) { var options = subscriptions.GetBatchOptions(id); writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); using (var cts = new CancellationTokenSource()) using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) { Etag lastProcessedDocEtag = null; var batchSize = 0; var batchDocCount = 0; var hasMoreDocs = false; var config = subscriptions.GetSubscriptionConfig(id); var startEtag = config.AckEtag; var criteria = config.Criteria; do { Database.TransactionalStorage.Batch(accessor => { // we may be sending a LOT of documents to the user, and most // of them aren't going to be relevant for other ops, so we are going to skip // the cache for that, to avoid filling it up very quickly using (DocumentCacher.SkipSettingDocumentsInDocumentCache()) { Database.Documents.GetDocuments(-1, options.MaxDocCount - batchDocCount, startEtag, cts.Token, doc => { timeout.Delay(); if (options.MaxSize.HasValue && batchSize >= options.MaxSize) { return; } if (batchDocCount >= options.MaxDocCount) { return; } lastProcessedDocEtag = doc.Etag; if (doc.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)) { return; } if (MatchCriteria(criteria, doc) == false) { return; } doc.ToJson().WriteTo(writer); writer.WriteRaw(Environment.NewLine); batchSize += doc.SerializedSizeOnDisk; batchDocCount++; }); } if (lastProcessedDocEtag == null) { hasMoreDocs = false; } else { var lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag(); hasMoreDocs = EtagUtil.IsGreaterThan(lastDocEtag, lastProcessedDocEtag); startEtag = lastProcessedDocEtag; } }); } while (hasMoreDocs && batchDocCount < options.MaxDocCount && (options.MaxSize.HasValue == false || batchSize < options.MaxSize)); writer.WriteEndArray(); if (batchDocCount > 0) { writer.WritePropertyName("LastProcessedEtag"); writer.WriteValue(lastProcessedDocEtag.ToString()); sentDocuments = true; } writer.WriteEndObject(); writer.Flush(); } } if (sentDocuments) { subscriptions.UpdateBatchSentTime(id); } }
public IEnumerable <JsonDocument> GetDocumentsAfter(Etag etag, int take, CancellationToken cancellationToken, long?maxSize = null, Etag untilEtag = null, TimeSpan?timeout = null) { if (take < 0) { throw new ArgumentException("must have zero or positive value", "take"); } if (take == 0) { yield break; } if (string.IsNullOrEmpty(etag)) { throw new ArgumentNullException("etag"); } Stopwatch duration = null; if (timeout != null) { duration = Stopwatch.StartNew(); } using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag) .Iterate(Snapshot, writeBatch.Value)) { var slice = (Slice)etag.ToString(); if (iterator.Seek(slice) == false) { yield break; } if (iterator.CurrentKey.Equals(slice)) // need gt, not ge { if (iterator.MoveNext() == false) { yield break; } } long fetchedDocumentTotalSize = 0; int fetchedDocumentCount = 0; do { cancellationToken.ThrowIfCancellationRequested(); var docEtag = Etag.Parse(iterator.CurrentKey.ToString()); if (untilEtag != null) { if (EtagUtil.IsGreaterThan(docEtag, untilEtag)) { yield break; } } var key = GetKeyFromCurrent(iterator); var document = DocumentByKey(key); if (document == null) //precaution - should never be true { throw new InvalidDataException(string.Format("Data corruption - the key = '{0}' was found in the documents indice, but matching document was not found", key)); } if (!document.Etag.Equals(docEtag)) { throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between document and its indice", key)); } fetchedDocumentTotalSize += document.SerializedSizeOnDisk; fetchedDocumentCount++; if (maxSize.HasValue && fetchedDocumentTotalSize >= maxSize) { yield return(document); yield break; } yield return(document); if (timeout != null) { if (duration.Elapsed > timeout.Value) { yield break; } } } while (iterator.MoveNext() && fetchedDocumentCount < take); } }
private async Task <bool> EnqueueMissingUpdatesAsync(ISynchronizationServerClient destinationSyncClient, SourceSynchronizationInformation synchronizationInfo, IList <FileHeader> needSyncingAgain) { LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}", needSyncingAgain); var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(synchronizationInfo.LastSourceFileEtag, NumberOfFilesToCheckForSynchronization), FileHeaderNameEqualityComparer.Instance); LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}", filesToSynchronization); foreach (FileHeader needSyncing in needSyncingAgain) { filesToSynchronization.Add(needSyncing); } var filteredFilesToSynchronization = filesToSynchronization.Where( x => synchronizationStrategy.Filter(x, synchronizationInfo.DestinationServerId, filesToSynchronization)).ToList(); if (filesToSynchronization.Count > 0) { LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization); } if (filteredFilesToSynchronization.Count == 0) { var lastFileBeforeFiltering = filesToSynchronization.LastOrDefault(); if (lastFileBeforeFiltering == null) { return(true); // there are no more files that need } if (lastFileBeforeFiltering.Etag == synchronizationInfo.LastSourceFileEtag) { return(true); // already updated etag on destination side } await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, lastFileBeforeFiltering.Etag).ConfigureAwait(false); return(false); // all docs has been filtered out, update etag on destination side and retry } var destinationUrl = destinationSyncClient.BaseUrl; bool enqueued = false; var maxEtagOfFilteredDoc = Etag.Empty; foreach (var fileHeader in filteredFilesToSynchronization) { context.CancellationToken.ThrowIfCancellationRequested(); var file = fileHeader.FullPath; var localMetadata = GetLocalMetadata(file); RavenJObject destinationMetadata; try { destinationMetadata = await destinationSyncClient.GetMetadataForAsync(file).ConfigureAwait(false); } catch (Exception ex) { Log.WarnException( string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file, destinationUrl), ex); continue; } NoSyncReason reason; var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason); if (work == null) { if (Log.IsDebugEnabled) { Log.Debug("File '{0}' was not synchronized to {1}. {2}", file, destinationUrl, reason.GetDescription()); } switch (reason) { case NoSyncReason.ContainedInDestinationHistory: case NoSyncReason.DestinationFileConflicted: case NoSyncReason.NoNeedToDeleteNonExistigFile: var localEtag = Etag.Parse(localMetadata.Value <string>(Constants.MetadataEtagField)); if (reason == NoSyncReason.ContainedInDestinationHistory) { RemoveSyncingConfiguration(file, destinationUrl); } else if (reason == NoSyncReason.DestinationFileConflicted) { if (needSyncingAgain.Contains(fileHeader, FileHeaderNameEqualityComparer.Instance) == false) { CreateSyncingConfiguration(fileHeader.Name, fileHeader.Etag, destinationUrl, SynchronizationType.Unknown); } } else if (reason == NoSyncReason.NoNeedToDeleteNonExistigFile) { // after the upgrade to newer build there can be still an existing syncing configuration for it RemoveSyncingConfiguration(file, destinationUrl); } if (EtagUtil.IsGreaterThan(localEtag, maxEtagOfFilteredDoc)) { maxEtagOfFilteredDoc = localEtag; } break; } continue; } if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) { publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Enqueue, Direction = SynchronizationDirection.Outgoing }); } enqueued = true; } if (enqueued == false && EtagUtil.IsGreaterThan(maxEtagOfFilteredDoc, synchronizationInfo.LastSourceFileEtag)) { await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, maxEtagOfFilteredDoc).ConfigureAwait(false); return(false); // we bumped the last synced etag on a destination server, let it know it need to repeat the operation } return(true); }
private void BackgroundSqlReplication() { int workCounter = 0; while (Database.WorkContext.DoWork) { IsRunning = !IsHotSpare() && !shouldPause; if (!IsRunning) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var config = GetConfiguredReplicationDestinations(); if (config.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var localReplicationStatus = GetReplicationStatus(); // remove all last replicated statuses which are not in the config UpdateLastReplicatedStatus(localReplicationStatus, config); var relevantConfigs = config.Where(x => { if (x.Disabled) { return(false); } var sqlReplicationStatistics = statistics.GetOrDefault(x.Name); if (sqlReplicationStatistics == null) { return(true); } return(SystemTime.UtcNow >= sqlReplicationStatistics.SuspendUntil); }) // have error or the timeout expired .ToList(); var configGroups = SqlReplicationClassifier.GroupConfigs(relevantConfigs, c => GetLastEtagFor(localReplicationStatus, c)); if (configGroups.Count == 0) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); continue; } var usedPrefetchers = new ConcurrentSet <PrefetchingBehavior>(); var groupedConfigs = configGroups .Select(x => { var result = new SqlConfigGroup { LastReplicatedEtag = x.Key, ConfigsToWorkOn = x.Value }; SetPrefetcherForIndexingGroup(result, usedPrefetchers); return(result); }) .ToList(); var successes = new ConcurrentQueue <Tuple <SqlReplicationConfigWithLastReplicatedEtag, Etag> >(); var waitForWork = new bool[groupedConfigs.Count]; try { BackgroundTaskExecuter.Instance.ExecuteAll(Database.WorkContext, groupedConfigs, (sqlConfigGroup, i) => { Database.WorkContext.CancellationToken.ThrowIfCancellationRequested(); var prefetchingBehavior = sqlConfigGroup.PrefetchingBehavior; var configsToWorkOn = sqlConfigGroup.ConfigsToWorkOn; List <JsonDocument> documents; var entityNamesToIndex = new HashSet <string>(configsToWorkOn.Select(x => x.RavenEntityName), StringComparer.OrdinalIgnoreCase); using (prefetchingBehavior.DocumentBatchFrom(sqlConfigGroup.LastReplicatedEtag, out documents, entityNamesToIndex)) { Etag latestEtag = null, lastBatchEtag = null; if (documents.Count != 0) { lastBatchEtag = documents[documents.Count - 1].Etag; } var replicationDuration = Stopwatch.StartNew(); documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here if (documents.Count != 0) { latestEtag = documents[documents.Count - 1].Etag; } documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false); var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >(); foreach (var configToWorkOn in configsToWorkOn) { var cfg = configToWorkOn; Database.TransactionalStorage.Batch(accessor => { deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg), cfg.LastReplicatedEtag, latestEtag, MaxNumberOfDeletionsToReplicate + 1) .ToList(); }); } // No documents AND there aren't any deletes to replicate if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0) { // so we filtered some documents, let us update the etag about that. if (latestEtag != null) { foreach (var configToWorkOn in configsToWorkOn) { successes.Enqueue(Tuple.Create(configToWorkOn, latestEtag)); } } else { waitForWork[i] = true; } return; } var itemsToReplicate = documents.Select(x => { JsonDocument.EnsureIdInMetadata(x); var doc = x.ToJson(); doc[Constants.DocumentIdFieldName] = x.Key; return(new ReplicatedDoc { Document = doc, Etag = x.Etag, Key = x.Key, SerializedSizeOnDisk = x.SerializedSizeOnDisk }); }).ToList(); try { BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, configsToWorkOn, replicationConfig => { try { var startTime = SystemTime.UtcNow; var spRepTime = new Stopwatch(); spRepTime.Start(); var lastReplicatedEtag = replicationConfig.LastReplicatedEtag; var deletedDocs = deletedDocsByConfig[replicationConfig]; var docsToReplicate = itemsToReplicate .Where(x => lastReplicatedEtag.CompareTo(x.Etag) < 0) // haven't replicate the etag yet .Where(document => { var info = Database.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0) { if (Log.IsDebugEnabled) { Log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, replicationConfig.Name, info.TouchedEtag); } return(false); } } return(true); }); if (deletedDocs.Count >= MaxNumberOfDeletionsToReplicate + 1) { docsToReplicate = docsToReplicate.Where(x => EtagUtil.IsGreaterThan(x.Etag, deletedDocs[deletedDocs.Count - 1].Etag) == false); } var docsToReplicateAsList = docsToReplicate.ToList(); var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicateAsList); if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicateAsList.Count == 0) { currentLatestEtag = lastBatchEtag; } int countOfReplicatedItems = 0; if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) && ReplicateChangesToDestination(replicationConfig, docsToReplicateAsList, out countOfReplicatedItems)) { if (deletedDocs.Count > 0) { Database.TransactionalStorage.Batch(accessor => accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag)); } successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag)); } spRepTime.Stop(); var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick); var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig); sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems); sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds); UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicateAsList.Count); } catch (Exception e) { Log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Exception = e.ToString(), Title = "Sql Replication failure to replication", Message = "Sql Replication could not replicate to " + replicationConfig.Name, UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name }); } }); } finally { prefetchingBehavior.CleanupDocuments(lastBatchEtag); prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed); } } }); if (successes.Count == 0) { if (waitForWork.All(x => x)) { Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication"); } continue; } foreach (var t in successes) { var cfg = t.Item1; var currentLatestEtag = t.Item2; //If a reset was requested we don't want to update the last replicated etag. //If we do register the success the reset will become a noop. bool isReset; if (ResetRequested.TryGetValue(t.Item1.Name, out isReset) && isReset) { continue; } var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase)); if (destEtag == null) { localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag { Name = cfg.Name, LastDocEtag = currentLatestEtag ?? Etag.Empty }); } else { var lastDocEtag = destEtag.LastDocEtag; if (currentLatestEtag != null && EtagUtil.IsGreaterThan(currentLatestEtag, lastDocEtag)) { lastDocEtag = currentLatestEtag; } destEtag.LastDocEtag = lastDocEtag; } } //We are done recording success for this batch so we can clear the reset dictionary ResetRequested.Clear(); SaveNewReplicationStatus(localReplicationStatus); } finally { AfterReplicationCompleted(successes.Count); RemoveUnusedPrefetchers(usedPrefetchers); } } }
private async Task <IEnumerable <Task <SynchronizationReport> > > SynchronizeDestinationAsync(SynchronizationDestination destination, bool forceSyncingAll) { ICredentials credentials = destination.Credentials; var destinationSyncClient = new SynchronizationServerClient(destination.ServerUrl, destination.FileSystem, destination.ApiKey, credentials); bool repeat; do { var lastETag = await destinationSyncClient.GetLastSynchronizationFromAsync(storage.Id).ConfigureAwait(false); var activeTasks = synchronizationQueue.Active; var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList(); var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationSyncClient).ConfigureAwait(false); var needSyncingAgain = new List <FileHeader>(); Debug.Assert(filesNeedConfirmation.Count == confirmations.Length); for (int i = 0; i < confirmations.Length; i++) { var confirmation = confirmations[i]; if (confirmation.Status == FileStatus.Safe) { if (Log.IsDebugEnabled) { Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName); } RemoveSyncingConfiguration(confirmation.FileName, destination.Url); } else { storage.Batch(accessor => { var fileHeader = accessor.ReadFile(confirmation.FileName); if (fileHeader == null) { if (Log.IsDebugEnabled) { Log.Debug("Destination server {0} said that file '{1}' is {2} but such file no longer exists. Removing related syncing configuration", destination, confirmation.FileName, confirmation.Status); } RemoveSyncingConfiguration(confirmation.FileName, destination.Url); } else if (EtagUtil.IsGreaterThan(fileHeader.Etag, filesNeedConfirmation[i].FileETag)) { if (Log.IsDebugEnabled) { Log.Debug("Destination server {0} said that file '{1}' is {2} but such file has been changed since we stored the syncing configuration. " + "Stored etag in configuration is {3} while current file etag is {4}. Removing related syncing configuration", destination, confirmation.FileName, confirmation.Status, fileHeader.Etag, filesNeedConfirmation[i].FileETag); } RemoveSyncingConfiguration(confirmation.FileName, destination.Url); } else { needSyncingAgain.Add(fileHeader); if (Log.IsDebugEnabled) { Log.Debug("Destination server {0} said that file '{1}' is {2}.", destination, confirmation.FileName, confirmation.Status); } } }); } } if (synchronizationQueue.NumberOfPendingSynchronizationsFor(destination.Url) < AvailableSynchronizationRequestsTo(destination.Url)) { repeat = await EnqueueMissingUpdatesAsync(destinationSyncClient, lastETag, needSyncingAgain).ConfigureAwait(false) == false; } else { repeat = false; } }while (repeat); return(SynchronizePendingFilesAsync(destinationSyncClient, forceSyncingAll)); }