public void GetDocumentsBatchFromShouldReturnUniqueResults2()
{
var prefetcher = CreatePrefetcher();
AddDocumentsToTransactionalStorage(prefetcher.TransactionalStorage, 100);
var documents = prefetcher.PrefetchingBehavior.GetDocumentsBatchFrom(Etag.Empty, 1);
var document = documents.Single();
AddDocumentResult result = null;
prefetcher.TransactionalStorage.Batch(accessor => result = accessor.Documents.AddDocument("keys/2", null, document.DataAsJson, document.Metadata));
documents = prefetcher.PrefetchingBehavior.GetDocumentsBatchFrom(document.Etag, 200);
Assert.Equal(99, documents.Count);
var prevEtag = Etag.Empty;
var keys = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (var doc in documents)
{
Assert.True(EtagUtil.IsGreaterThan(document.Etag, prevEtag));
Assert.True(keys.Add(doc.Key));
}
}
public Etag GetBestNextDocumentEtag(Etag etag)
{
if (etag == null)
{
throw new ArgumentNullException("etag");
}
using (var iter = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
.Iterate(Snapshot, writeBatch.Value))
{
if (!iter.Seek((Slice)etag.ToString()) &&
!iter.Seek(Slice.BeforeAllKeys)) //if parameter etag not found, scan from beginning. if empty --> return original etag
{
return(etag);
}
do
{
var docEtag = Etag.Parse(iter.CurrentKey.ToString());
if (EtagUtil.IsGreaterThan(docEtag, etag))
{
return(docEtag);
}
} while (iter.MoveNext());
}
return(etag); //if not found, return the original etag
}
private bool IsIndexStateValid(LuceneDirectory luceneDirectory)
{
// 1. If commitData is null it means that there were no commits, so just in case we are resetting to Etag.Empty
// 2. If no 'LastEtag' in commitData then we consider it an invalid index
// 3. If 'LastEtag' is present (and valid), then resetting to it (if it is lower than lastStoredEtag)
var commitData = IndexReader.GetCommitUserData(luceneDirectory);
string value;
Etag lastEtag = null;
if (commitData != null && commitData.TryGetValue("LastEtag", out value))
{
Etag.TryParse(value, out lastEtag); // etag will be null if parsing will fail
}
var lastStoredEtag = GetLastEtagForIndex() ?? Etag.Empty;
lastEtag = lastEtag ?? Etag.Empty;
if (EtagUtil.IsGreaterThan(lastEtag, lastStoredEtag))
{
return(false);
}
var checkIndex = new CheckIndex(luceneDirectory);
var status = checkIndex.CheckIndex_Renamed_Method();
return(status.clean);
}
public void AbstractScriptedIndexCreationTaskWillResetIndexIfDocumentHasChanged()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new Person
{
Name = "Name1"
});
session.SaveChanges();
}
new People_By_Name().Execute(store);
var index = new People_By_Name_With_Scripts();
store.DatabaseCommands.Put(ScriptedIndexResults.IdPrefix + index.IndexName, null, new RavenJObject(), new RavenJObject());
WaitForIndexing(store);
var stats = store.DatabaseCommands.GetStatistics();
var indexStats = stats.Indexes.First(x => x.Name == index.IndexName);
Assert.True(EtagUtil.IsGreaterThan(indexStats.LastIndexedEtag, Etag.Empty));
store.DatabaseCommands.Admin.StopIndexing();
index.Execute(store);
stats = store.DatabaseCommands.GetStatistics();
indexStats = stats.Indexes.First(x => x.Name == index.IndexName);
Assert.True(indexStats.LastIndexedEtag.Equals(Etag.Empty));
}
}
public void AbstractScriptedIndexCreationTaskWillNotResetIndexIfNothingHasChanged()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new Person
{
Name = "Name1"
});
session.SaveChanges();
}
var index = new People_By_Name_With_Scripts();
index.Execute(store);
WaitForIndexing(store);
var stats = store.DatabaseCommands.GetStatistics();
var indexStats = stats.Indexes.First(x => x.Name == index.IndexName);
var lastIndexedEtag = indexStats.LastIndexedEtag;
Assert.True(EtagUtil.IsGreaterThan(lastIndexedEtag, Etag.Empty));
store.DatabaseCommands.Admin.StopIndexing();
index.Execute(store);
stats = store.DatabaseCommands.GetStatistics();
indexStats = stats.Indexes.First(x => x.Name == index.IndexName);
Assert.True(indexStats.LastIndexedEtag.Equals(lastIndexedEtag) ||
EtagUtil.IsGreaterThan(indexStats.LastIndexedEtag, lastIndexedEtag));
}
}
private void SaveSynchronizationSourceInformation(FileSystemInfo sourceFileSystem, Etag lastSourceEtag)
{
var lastSynchronizationInformation = GetLastSynchronization(sourceFileSystem.Id);
if (EtagUtil.IsGreaterThan(lastSynchronizationInformation.LastSourceFileEtag, lastSourceEtag))
{
return;
}
var synchronizationSourceInfo = new SourceSynchronizationInformation
{
LastSourceFileEtag = lastSourceEtag,
SourceServerUrl = sourceFileSystem.Url,
DestinationServerId = Storage.Id
};
var key = SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + sourceFileSystem.Id;
Storage.Batch(accessor => accessor.SetConfig(key, JsonExtensions.ToJObject(synchronizationSourceInfo)));
if (Log.IsDebugEnabled)
{
Log.Debug("Saved last synchronized file ETag {0} from {1} ({2})", lastSourceEtag, sourceFileSystem.Url, sourceFileSystem.Id);
}
}
public void GetDocumentsBatchFromShouldReturnResultsOrderedByEtag()
{
var prefetcher = CreatePrefetcher();
AddDocumentsToTransactionalStorage(prefetcher.TransactionalStorage, 100);
var documents = prefetcher.PrefetchingBehavior.GetDocumentsBatchFrom(Etag.Empty, 100);
var prevEtag = Etag.Empty;
foreach (var document in documents)
{
Assert.True(EtagUtil.IsGreaterThan(document.Etag, prevEtag));
prevEtag = document.Etag;
}
}
public IEnumerable <JsonDocument> GetDocumentsAfter(Etag etag, int take, CancellationToken cancellationToken, long?maxSize = null, Etag untilEtag = null, TimeSpan?timeout = null)
{
Api.JetSetCurrentIndex(session, Documents, "by_etag");
Api.MakeKey(session, Documents, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Documents, SeekGrbit.SeekGT) == false)
{
yield break;
}
long totalSize = 0;
int count = 0;
Stopwatch duration = null;
if (timeout != null)
{
duration = Stopwatch.StartNew();
}
do
{
cancellationToken.ThrowIfCancellationRequested();
if (untilEtag != null && count > 0)
{
var docEtag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"]));
if (EtagUtil.IsGreaterThan(docEtag, untilEtag))
{
yield break;
}
}
var readCurrentDocument = ReadCurrentDocument();
totalSize += readCurrentDocument.SerializedSizeOnDisk;
if (maxSize != null && totalSize > maxSize.Value)
{
yield return(readCurrentDocument);
yield break;
}
yield return(readCurrentDocument);
count++;
if (timeout != null)
{
if (duration.Elapsed > timeout.Value)
{
yield break;
}
}
} while (Api.TryMoveNext(session, Documents) && count < take);
}
public IEnumerable <JsonDocument> GetDocumentsAfterWithIdStartingWith(Etag etag, string idPrefix, int take, CancellationToken cancellationToken, long?maxSize = null, Etag untilEtag = null, TimeSpan?timeout = null, Action <Etag> lastProcessedDocument = null,
Reference <bool> earlyExit = null)
{
if (earlyExit != null)
{
earlyExit.Value = false;
}
if (take < 0)
{
throw new ArgumentException("must have zero or positive value", "take");
}
if (take == 0)
{
yield break;
}
if (string.IsNullOrEmpty(etag))
{
throw new ArgumentNullException("etag");
}
Stopwatch duration = null;
if (timeout != null)
{
duration = Stopwatch.StartNew();
}
Etag lastDocEtag = null;
using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
.Iterate(Snapshot, writeBatch.Value))
{
var slice = (Slice)etag.ToString();
if (iterator.Seek(slice) == false)
{
yield break;
}
if (iterator.CurrentKey.Equals(slice)) // need gt, not ge
{
if (iterator.MoveNext() == false)
{
yield break;
}
}
long fetchedDocumentTotalSize = 0;
int fetchedDocumentCount = 0;
Etag docEtag = etag;
do
{
cancellationToken.ThrowIfCancellationRequested();
docEtag = Etag.Parse(iterator.CurrentKey.ToString());
// We can skip many documents so the timeout should be at the start of the process to be executed.
if (timeout != null)
{
if (duration.Elapsed > timeout.Value)
{
if (earlyExit != null)
{
earlyExit.Value = true;
}
break;
}
}
if (untilEtag != null)
{
// This is not a failure, we are just ahead of when we expected to.
if (EtagUtil.IsGreaterThan(docEtag, untilEtag))
{
break;
}
}
var key = GetKeyFromCurrent(iterator);
if (!string.IsNullOrEmpty(idPrefix))
{
if (!key.StartsWith(idPrefix, StringComparison.OrdinalIgnoreCase))
{
// We assume that we have processed it because it is not of our interest.
lastDocEtag = docEtag;
continue;
}
}
var document = DocumentByKey(key);
if (document == null) //precaution - should never be true
{
if (SkipConsistencyCheck)
{
continue;
}
throw new InvalidDataException(string.Format("Data corruption - the key = '{0}' was found in the documents index, but matching document was not found", key));
}
if (!document.Etag.Equals(docEtag) && !SkipConsistencyCheck)
{
throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between document and its index", key));
}
fetchedDocumentTotalSize += document.SerializedSizeOnDisk;
fetchedDocumentCount++;
yield return(document);
lastDocEtag = docEtag;
if (maxSize.HasValue && fetchedDocumentTotalSize >= maxSize)
{
if (untilEtag != null && earlyExit != null)
{
earlyExit.Value = true;
}
break;
}
if (fetchedDocumentCount >= take)
{
if (untilEtag != null && earlyExit != null)
{
earlyExit.Value = true;
}
break;
}
} while (iterator.MoveNext());
}
// We notify the last that we considered.
if (lastProcessedDocument != null)
{
lastProcessedDocument(lastDocEtag);
}
}
public HttpResponseMessage ExplainGet(string docId)
{
if (string.IsNullOrEmpty(docId))
{
return(GetMessageWithString("Document key is required.", HttpStatusCode.BadRequest));
}
var destinationUrl = GetQueryStringValue("destinationUrl");
if (string.IsNullOrEmpty(destinationUrl))
{
return(GetMessageWithString("Destination url is required.", HttpStatusCode.BadRequest));
}
var databaseName = GetQueryStringValue("databaseName");
if (string.IsNullOrEmpty(databaseName))
{
return(GetMessageWithString("Destination database name is required.", HttpStatusCode.BadRequest));
}
var result = new ReplicationExplanationForDocument
{
Key = docId,
Destination = new ReplicationExplanationForDocument.DestinationInformation
{
Url = destinationUrl,
DatabaseName = databaseName
}
};
var destinations = ReplicationTask.GetReplicationDestinations(x => string.Equals(x.Url, destinationUrl, StringComparison.OrdinalIgnoreCase) && string.Equals(x.Database, databaseName, StringComparison.OrdinalIgnoreCase));
if (destinations == null || destinations.Length == 0)
{
result.Message = string.Format("Could not find replication destination for a given url ('{0}') and database ('{1}').", destinationUrl, databaseName);
return(GetMessageWithObject(result));
}
if (destinations.Length > 1)
{
result.Message = string.Format("There is more than one replication destination for a given url ('{0}') and database ('{1}').", destinationUrl, databaseName);
return(GetMessageWithObject(result));
}
var destination = destinations[0];
var destinationsReplicationInformationForSource = ReplicationTask.GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
{
result.Message = "Could not connect to destination server.";
return(GetMessageWithObject(result));
}
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
result.Destination.ServerInstanceId = destinationId;
result.Destination.LastDocumentEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
var document = Database.Documents.Get(docId, null);
if (document == null)
{
result.Message = string.Format("Document with given key ('{0}') does not exist.", docId);
return(GetMessageWithObject(result));
}
result.Key = document.Key;
result.Etag = document.Etag;
string reason;
if (destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) == false)
{
result.Message = reason;
return(GetMessageWithObject(result));
}
reason = EtagUtil.IsGreaterThan(document.Etag, destinationsReplicationInformationForSource.LastDocumentEtag) ? "Document will be replicated." : "Document should have been replicated.";
result.Message = reason;
return(GetMessageWithObject(result));
}
public IEnumerable <JsonDocument> GetDocumentsAfterWithIdStartingWith(Etag etag, string idPrefix, int take, CancellationToken cancellationToken, long?maxSize = null, Etag untilEtag = null, TimeSpan?timeout = null)
{
Api.JetSetCurrentIndex(session, Documents, "by_etag");
Api.MakeKey(session, Documents, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Documents, SeekGrbit.SeekGT) == false)
{
yield break;
}
long totalSize = 0;
int count = 0;
Stopwatch duration = null;
if (timeout != null)
{
duration = Stopwatch.StartNew();
}
do
{
cancellationToken.ThrowIfCancellationRequested();
// We can skip many documents so the timeout should be at the start of the process to be executed.
if (timeout != null)
{
if (duration.Elapsed > timeout.Value)
{
yield break;
}
}
if (untilEtag != null && count > 0)
{
var docEtag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"]));
if (EtagUtil.IsGreaterThan(docEtag, untilEtag))
{
yield break;
}
}
var key = Api.RetrieveColumnAsString(session, Documents, tableColumnsCache.DocumentsColumns["key"], Encoding.Unicode);
if (!string.IsNullOrEmpty(idPrefix))
{
// Check if the ignore case is correct.
if (!key.StartsWith(idPrefix, StringComparison.OrdinalIgnoreCase))
{
continue;
}
}
var readCurrentDocument = ReadCurrentDocument(key);
totalSize += readCurrentDocument.SerializedSizeOnDisk;
yield return(readCurrentDocument);
if (maxSize != null && totalSize > maxSize.Value)
{
yield break;
}
count++;
} while (Api.TryMoveNext(session, Documents) && count < take);
}
public IEnumerable <AttachmentInformation> GetAttachmentsAfter(Etag value, int take, long maxTotalSize)
{
if (take < 0)
{
throw new ArgumentException("must have zero or positive value", "take");
}
if (maxTotalSize < 0)
{
throw new ArgumentException("must have zero or positive value", "maxTotalSize");
}
if (take == 0)
{
yield break; //edge case
}
using (var iter = attachmentsTable.GetIndex(Tables.Attachments.Indices.ByEtag)
.Iterate(Snapshot, writeBatch.Value))
{
if (!iter.Seek(Slice.BeforeAllKeys))
{
yield break;
}
var fetchedDocumentCount = 0;
var fetchedTotalSize = 0;
do
{
if (iter.CurrentKey == null || iter.CurrentKey.Equals(Slice.Empty))
{
yield break;
}
var attachmentEtag = Etag.Parse(iter.CurrentKey.ToString());
if (!EtagUtil.IsGreaterThan(attachmentEtag, value))
{
continue;
}
string key;
using (var keyStream = iter.CreateReaderForCurrent().AsStream())
key = keyStream.ReadStringWithoutPrefix();
var attachmentInfo = AttachmentInfoByKey(key);
if (!attachmentInfo.Etag.Equals(attachmentEtag))
{
throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between attachment and its indice", key));
}
fetchedTotalSize += attachmentInfo.Size;
fetchedDocumentCount++;
if (fetchedTotalSize > maxTotalSize)
{
yield break;
}
if (fetchedDocumentCount >= take || fetchedTotalSize == maxTotalSize)
{
yield return(attachmentInfo);
yield break;
}
yield return(attachmentInfo);
} while (iter.MoveNext());
}
}
private void BackgroundSqlReplication()
{
int workCounter = 0;
while (Database.WorkContext.DoWork)
{
IsRunning = !shouldPause;
if (!IsRunning)
{
continue;
}
var config = GetConfiguredReplicationDestinations();
if (config.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var localReplicationStatus = GetReplicationStatus();
var relevantConfigs = config.Where(x =>
{
if (x.Disabled)
{
return(false);
}
var sqlReplicationStatistics = statistics.GetOrDefault(x.Name);
if (sqlReplicationStatistics == null)
{
return(true);
}
return(SystemTime.UtcNow >= sqlReplicationStatistics.SuspendUntil);
}) // have error or the timeout expired
.ToList();
if (relevantConfigs.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus);
if (leastReplicatedEtag == null)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
List <JsonDocument> documents;
using (prefetchingBehavior.DocumentBatchFrom(leastReplicatedEtag, out documents))
{
Etag latestEtag = null, lastBatchEtag = null;
if (documents.Count != 0)
{
lastBatchEtag = documents[documents.Count - 1].Etag;
}
var replicationDuration = Stopwatch.StartNew();
documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here
if (documents.Count != 0)
{
latestEtag = documents[documents.Count - 1].Etag;
}
documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false);
var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >();
foreach (var relevantConfig in relevantConfigs)
{
var cfg = relevantConfig;
Database.TransactionalStorage.Batch(accessor =>
{
deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg),
GetLastEtagFor(localReplicationStatus, cfg),
latestEtag,
MaxNumberOfDeletionsToReplicate + 1)
.ToList();
});
}
// No documents AND there aren't any deletes to replicate
if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0)
{
if (latestEtag != null)
{
// so we filtered some documents, let us update the etag about that.
foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags)
{
if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0)
{
lastReplicatedEtag.LastDocEtag = latestEtag;
}
}
latestEtag = Etag.Max(latestEtag, lastBatchEtag);
SaveNewReplicationStatus(localReplicationStatus);
}
else // no point in waiting if we just saved a new doc
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
}
continue;
}
var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >();
try
{
var itemsToReplicate = documents.Select(x =>
{
JsonDocument.EnsureIdInMetadata(x);
var doc = x.ToJson();
doc[Constants.DocumentIdFieldName] = x.Key;
return(new ReplicatedDoc
{
Document = doc,
Etag = x.Etag,
Key = x.Key,
SerializedSizeOnDisk = x.SerializedSizeOnDisk
});
}).ToList();
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig =>
{
try
{
var startTime = SystemTime.UtcNow;
Stopwatch spRepTime = new Stopwatch();
spRepTime.Start();
var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig);
var deletedDocs = deletedDocsByConfig[replicationConfig];
var docsToReplicate = itemsToReplicate
.Where(x => lastReplicatedEtag.CompareTo(x.Etag) < 0) // haven't replicate the etag yet
.Where(document =>
{
var info = Database.Documents.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0)
{
log.Debug(
"Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
document.Key, replicationConfig.Name, info.TouchedEtag);
return(false);
}
}
return(true);
});
if (deletedDocs.Count >= MaxNumberOfDeletionsToReplicate + 1)
{
docsToReplicate = docsToReplicate.Where(x => EtagUtil.IsGreaterThan(x.Etag, deletedDocs[deletedDocs.Count - 1].Etag) == false);
}
var docsToReplicateAsList = docsToReplicate.ToList();
var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicateAsList);
if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicateAsList.Count == 0)
{
currentLatestEtag = lastBatchEtag;
}
int countOfReplicatedItems = 0;
if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) &&
ReplicateChangesToDestination(replicationConfig, docsToReplicateAsList, out countOfReplicatedItems))
{
if (deletedDocs.Count > 0)
{
Database.TransactionalStorage.Batch(accessor =>
accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag));
}
successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag));
}
spRepTime.Stop();
var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick);
var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig);
sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems);
sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems);
sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds);
UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicateAsList.Count);
}
catch (Exception e)
{
log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Exception = e.ToString(),
Title = "Sql Replication failure to replication",
Message = "Sql Replication could not replicate to " + replicationConfig.Name,
UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name
});
}
});
if (successes.Count == 0)
{
continue;
}
foreach (var t in successes)
{
var cfg = t.Item1;
var currentLatestEtag = t.Item2;
var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
if (destEtag == null)
{
localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
{
Name = cfg.Name,
LastDocEtag = currentLatestEtag ?? Etag.Empty
});
}
else
{
var lastDocEtag = destEtag.LastDocEtag;
if (currentLatestEtag != null && EtagUtil.IsGreaterThan(currentLatestEtag, lastDocEtag))
{
lastDocEtag = currentLatestEtag;
}
destEtag.LastDocEtag = lastDocEtag;
}
}
SaveNewReplicationStatus(localReplicationStatus);
}
finally
{
AfterReplicationCompleted(successes.Count);
var min = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray()));
if (min != null)
{
var lastMinReplicatedEtag = min.ToEtag();
prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag);
prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed);
}
}
}
}
}
public IEnumerable <JsonDocument> GetDocumentsAfterWithIdStartingWith(Etag etag, string idPrefix, int take, CancellationToken cancellationToken, long?maxSize = null, Etag untilEtag = null, TimeSpan?timeout = null, Action <Etag> lastProcessedDocument = null,
Reference <bool> earlyExit = null)
{
if (earlyExit != null)
{
earlyExit.Value = false;
}
Api.JetSetCurrentIndex(session, Documents, "by_etag");
Api.MakeKey(session, Documents, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Documents, SeekGrbit.SeekGT) == false)
{
yield break;
}
long totalSize = 0;
int fetchedDocumentCount = 0;
Stopwatch duration = null;
if (timeout != null)
{
duration = Stopwatch.StartNew();
}
Etag lastDocEtag = null;
Etag docEtag = etag;
do
{
cancellationToken.ThrowIfCancellationRequested();
docEtag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"]));
// We can skip many documents so the timeout should be at the start of the process to be executed.
if (timeout != null)
{
if (duration.Elapsed > timeout.Value)
{
if (earlyExit != null)
{
earlyExit.Value = true;
}
break;
}
}
if (untilEtag != null && fetchedDocumentCount > 0)
{
// This is not a failure, we are just ahead of when we expected to.
if (EtagUtil.IsGreaterThan(docEtag, untilEtag))
{
break;
}
}
var key = Api.RetrieveColumnAsString(session, Documents, tableColumnsCache.DocumentsColumns["key"], Encoding.Unicode);
if (!string.IsNullOrEmpty(idPrefix))
{
if (!key.StartsWith(idPrefix, StringComparison.OrdinalIgnoreCase))
{
// We assume that we have processed it because it is not of our interest.
lastDocEtag = docEtag;
continue;
}
}
var readCurrentDocument = ReadCurrentDocument(key);
totalSize += readCurrentDocument.SerializedSizeOnDisk;
fetchedDocumentCount++;
yield return(readCurrentDocument);
lastDocEtag = docEtag;
if (maxSize != null && totalSize > maxSize.Value)
{
if (untilEtag != null && earlyExit != null)
{
earlyExit.Value = true;
}
break;
}
if (fetchedDocumentCount >= take)
{
if (untilEtag != null && earlyExit != null)
{
earlyExit.Value = true;
}
break;
}
}while (Api.TryMoveNext(session, Documents));
// We notify the last that we considered.
if (lastProcessedDocument != null)
{
lastProcessedDocument(lastDocEtag);
}
}
private void StreamToClient(long id, SubscriptionActions subscriptions, Stream stream)
{
var sentDocuments = false;
var bufferStream = new BufferedStream(stream, 1024 * 64);
using (var writer = new JsonTextWriter(new StreamWriter(bufferStream)))
{
var options = subscriptions.GetBatchOptions(id);
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout))
{
Etag lastProcessedDocEtag = null;
var batchSize = 0;
var batchDocCount = 0;
var processedDocuments = 0;
var hasMoreDocs = false;
var config = subscriptions.GetSubscriptionConfig(id);
var startEtag = config.AckEtag;
var criteria = config.Criteria;
bool isPrefixCriteria = !string.IsNullOrWhiteSpace(criteria.KeyStartsWith);
Func <JsonDocument, bool> addDocument = doc =>
{
timeout.Delay();
if (doc == null)
{
// we only have this heartbeat when the streaming has gone on for a long time
// and we haven't send anything to the user in a while (because of filtering, skipping, etc).
writer.WriteRaw(Environment.NewLine);
writer.Flush();
return(true);
}
processedDocuments++;
// We cant continue because we have already maxed out the batch bytes size.
if (options.MaxSize.HasValue && batchSize >= options.MaxSize)
{
return(false);
}
// We cant continue because we have already maxed out the amount of documents to send.
if (batchDocCount >= options.MaxDocCount)
{
return(false);
}
// We can continue because we are ignoring system documents.
if (doc.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase))
{
return(true);
}
// We can continue because we are ignoring the document as it doesn't fit the criteria.
if (MatchCriteria(criteria, doc) == false)
{
return(true);
}
doc.ToJson().WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
batchSize += doc.SerializedSizeOnDisk;
batchDocCount++;
return(true); // We get the next document
};
int retries = 0;
do
{
int lastIndex = processedDocuments;
Database.TransactionalStorage.Batch(accessor =>
{
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache())
{
if (isPrefixCriteria)
{
// If we don't get any document from GetDocumentsWithIdStartingWith it could be that we are in presence of a lagoon of uninteresting documents, so we are hitting a timeout.
lastProcessedDocEtag = Database.Documents.GetDocumentsWithIdStartingWith(criteria.KeyStartsWith, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument);
hasMoreDocs = false;
}
else
{
// It doesn't matter if we match the criteria or not, the document has been already processed.
lastProcessedDocEtag = Database.Documents.GetDocuments(-1, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument);
// If we don't get any document from GetDocuments it may be a signal that something is wrong.
if (lastProcessedDocEtag == null)
{
hasMoreDocs = false;
}
else
{
var lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
hasMoreDocs = EtagUtil.IsGreaterThan(lastDocEtag, lastProcessedDocEtag);
startEtag = lastProcessedDocEtag;
}
retries = lastIndex == batchDocCount ? retries : 0;
}
}
});
if (lastIndex == processedDocuments)
{
if (retries == 3)
{
log.Warn("Subscription processing did not end up replicating any documents for 3 times in a row, stopping operation", retries);
}
else
{
log.Warn("Subscription processing did not end up replicating any documents, due to possible storage error, retry number: {0}", retries);
}
retries++;
}
}while (retries < 3 && hasMoreDocs && batchDocCount < options.MaxDocCount && (options.MaxSize.HasValue == false || batchSize < options.MaxSize));
writer.WriteEndArray();
if (batchDocCount > 0 || isPrefixCriteria)
{
writer.WritePropertyName("LastProcessedEtag");
writer.WriteValue(lastProcessedDocEtag.ToString());
sentDocuments = true;
}
writer.WriteEndObject();
writer.Flush();
bufferStream.Flush();
}
}
if (sentDocuments)
{
subscriptions.UpdateBatchSentTime(id);
}
}
private void StreamToClient(long id, SubscriptionActions subscriptions, Stream stream)
{
var sentDocuments = false;
using (var streamWriter = new StreamWriter(stream))
using (var writer = new JsonTextWriter(streamWriter))
{
var options = subscriptions.GetBatchOptions(id);
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout))
{
Etag lastProcessedDocEtag = null;
var batchSize = 0;
var batchDocCount = 0;
var hasMoreDocs = false;
var config = subscriptions.GetSubscriptionConfig(id);
var startEtag = config.AckEtag;
var criteria = config.Criteria;
do
{
Database.TransactionalStorage.Batch(accessor =>
{
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
{
Database.Documents.GetDocuments(-1, options.MaxDocCount - batchDocCount, startEtag, cts.Token, doc =>
{
timeout.Delay();
if (options.MaxSize.HasValue && batchSize >= options.MaxSize)
{
return;
}
if (batchDocCount >= options.MaxDocCount)
{
return;
}
lastProcessedDocEtag = doc.Etag;
if (doc.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase))
{
return;
}
if (MatchCriteria(criteria, doc) == false)
{
return;
}
doc.ToJson().WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
batchSize += doc.SerializedSizeOnDisk;
batchDocCount++;
});
}
if (lastProcessedDocEtag == null)
{
hasMoreDocs = false;
}
else
{
var lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
hasMoreDocs = EtagUtil.IsGreaterThan(lastDocEtag, lastProcessedDocEtag);
startEtag = lastProcessedDocEtag;
}
});
} while (hasMoreDocs && batchDocCount < options.MaxDocCount && (options.MaxSize.HasValue == false || batchSize < options.MaxSize));
writer.WriteEndArray();
if (batchDocCount > 0)
{
writer.WritePropertyName("LastProcessedEtag");
writer.WriteValue(lastProcessedDocEtag.ToString());
sentDocuments = true;
}
writer.WriteEndObject();
writer.Flush();
}
}
if (sentDocuments)
{
subscriptions.UpdateBatchSentTime(id);
}
}
public IEnumerable <JsonDocument> GetDocumentsAfter(Etag etag, int take, CancellationToken cancellationToken, long?maxSize = null, Etag untilEtag = null, TimeSpan?timeout = null)
{
if (take < 0)
{
throw new ArgumentException("must have zero or positive value", "take");
}
if (take == 0)
{
yield break;
}
if (string.IsNullOrEmpty(etag))
{
throw new ArgumentNullException("etag");
}
Stopwatch duration = null;
if (timeout != null)
{
duration = Stopwatch.StartNew();
}
using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
.Iterate(Snapshot, writeBatch.Value))
{
var slice = (Slice)etag.ToString();
if (iterator.Seek(slice) == false)
{
yield break;
}
if (iterator.CurrentKey.Equals(slice)) // need gt, not ge
{
if (iterator.MoveNext() == false)
{
yield break;
}
}
long fetchedDocumentTotalSize = 0;
int fetchedDocumentCount = 0;
do
{
cancellationToken.ThrowIfCancellationRequested();
var docEtag = Etag.Parse(iterator.CurrentKey.ToString());
if (untilEtag != null)
{
if (EtagUtil.IsGreaterThan(docEtag, untilEtag))
{
yield break;
}
}
var key = GetKeyFromCurrent(iterator);
var document = DocumentByKey(key);
if (document == null) //precaution - should never be true
{
throw new InvalidDataException(string.Format("Data corruption - the key = '{0}' was found in the documents indice, but matching document was not found", key));
}
if (!document.Etag.Equals(docEtag))
{
throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between document and its indice", key));
}
fetchedDocumentTotalSize += document.SerializedSizeOnDisk;
fetchedDocumentCount++;
if (maxSize.HasValue && fetchedDocumentTotalSize >= maxSize)
{
yield return(document);
yield break;
}
yield return(document);
if (timeout != null)
{
if (duration.Elapsed > timeout.Value)
{
yield break;
}
}
} while (iterator.MoveNext() && fetchedDocumentCount < take);
}
}
private async Task <bool> EnqueueMissingUpdatesAsync(ISynchronizationServerClient destinationSyncClient,
SourceSynchronizationInformation synchronizationInfo,
IList <FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(synchronizationInfo.LastSourceFileEtag, NumberOfFilesToCheckForSynchronization),
FileHeaderNameEqualityComparer.Instance);
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization = filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, synchronizationInfo.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);
}
if (filteredFilesToSynchronization.Count == 0)
{
var lastFileBeforeFiltering = filesToSynchronization.LastOrDefault();
if (lastFileBeforeFiltering == null)
{
return(true); // there are no more files that need
}
if (lastFileBeforeFiltering.Etag == synchronizationInfo.LastSourceFileEtag)
{
return(true); // already updated etag on destination side
}
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, lastFileBeforeFiltering.Etag).ConfigureAwait(false);
return(false); // all docs has been filtered out, update etag on destination side and retry
}
var destinationUrl = destinationSyncClient.BaseUrl;
bool enqueued = false;
var maxEtagOfFilteredDoc = Etag.Empty;
foreach (var fileHeader in filteredFilesToSynchronization)
{
context.CancellationToken.ThrowIfCancellationRequested();
var file = fileHeader.FullPath;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationSyncClient.GetMetadataForAsync(file).ConfigureAwait(false);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
destinationUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", file, destinationUrl, reason.GetDescription());
}
switch (reason)
{
case NoSyncReason.ContainedInDestinationHistory:
case NoSyncReason.DestinationFileConflicted:
case NoSyncReason.NoNeedToDeleteNonExistigFile:
var localEtag = Etag.Parse(localMetadata.Value <string>(Constants.MetadataEtagField));
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
RemoveSyncingConfiguration(file, destinationUrl);
}
else if (reason == NoSyncReason.DestinationFileConflicted)
{
if (needSyncingAgain.Contains(fileHeader, FileHeaderNameEqualityComparer.Instance) == false)
{
CreateSyncingConfiguration(fileHeader.Name, fileHeader.Etag, destinationUrl, SynchronizationType.Unknown);
}
}
else if (reason == NoSyncReason.NoNeedToDeleteNonExistigFile)
{
// after the upgrade to newer build there can be still an existing syncing configuration for it
RemoveSyncingConfiguration(file, destinationUrl);
}
if (EtagUtil.IsGreaterThan(localEtag, maxEtagOfFilteredDoc))
{
maxEtagOfFilteredDoc = localEtag;
}
break;
}
continue;
}
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
enqueued = true;
}
if (enqueued == false && EtagUtil.IsGreaterThan(maxEtagOfFilteredDoc, synchronizationInfo.LastSourceFileEtag))
{
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, maxEtagOfFilteredDoc).ConfigureAwait(false);
return(false); // we bumped the last synced etag on a destination server, let it know it need to repeat the operation
}
return(true);
}
private void BackgroundSqlReplication()
{
int workCounter = 0;
while (Database.WorkContext.DoWork)
{
IsRunning = !IsHotSpare() && !shouldPause;
if (!IsRunning)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var config = GetConfiguredReplicationDestinations();
if (config.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var localReplicationStatus = GetReplicationStatus();
// remove all last replicated statuses which are not in the config
UpdateLastReplicatedStatus(localReplicationStatus, config);
var relevantConfigs = config.Where(x =>
{
if (x.Disabled)
{
return(false);
}
var sqlReplicationStatistics = statistics.GetOrDefault(x.Name);
if (sqlReplicationStatistics == null)
{
return(true);
}
return(SystemTime.UtcNow >= sqlReplicationStatistics.SuspendUntil);
}) // have error or the timeout expired
.ToList();
var configGroups = SqlReplicationClassifier.GroupConfigs(relevantConfigs, c => GetLastEtagFor(localReplicationStatus, c));
if (configGroups.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var usedPrefetchers = new ConcurrentSet <PrefetchingBehavior>();
var groupedConfigs = configGroups
.Select(x =>
{
var result = new SqlConfigGroup
{
LastReplicatedEtag = x.Key,
ConfigsToWorkOn = x.Value
};
SetPrefetcherForIndexingGroup(result, usedPrefetchers);
return(result);
})
.ToList();
var successes = new ConcurrentQueue <Tuple <SqlReplicationConfigWithLastReplicatedEtag, Etag> >();
var waitForWork = new bool[groupedConfigs.Count];
try
{
BackgroundTaskExecuter.Instance.ExecuteAll(Database.WorkContext, groupedConfigs, (sqlConfigGroup, i) =>
{
Database.WorkContext.CancellationToken.ThrowIfCancellationRequested();
var prefetchingBehavior = sqlConfigGroup.PrefetchingBehavior;
var configsToWorkOn = sqlConfigGroup.ConfigsToWorkOn;
List <JsonDocument> documents;
var entityNamesToIndex = new HashSet <string>(configsToWorkOn.Select(x => x.RavenEntityName), StringComparer.OrdinalIgnoreCase);
using (prefetchingBehavior.DocumentBatchFrom(sqlConfigGroup.LastReplicatedEtag, out documents, entityNamesToIndex))
{
Etag latestEtag = null, lastBatchEtag = null;
if (documents.Count != 0)
{
lastBatchEtag = documents[documents.Count - 1].Etag;
}
var replicationDuration = Stopwatch.StartNew();
documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here
if (documents.Count != 0)
{
latestEtag = documents[documents.Count - 1].Etag;
}
documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false);
var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >();
foreach (var configToWorkOn in configsToWorkOn)
{
var cfg = configToWorkOn;
Database.TransactionalStorage.Batch(accessor =>
{
deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg),
cfg.LastReplicatedEtag,
latestEtag,
MaxNumberOfDeletionsToReplicate + 1)
.ToList();
});
}
// No documents AND there aren't any deletes to replicate
if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0)
{
// so we filtered some documents, let us update the etag about that.
if (latestEtag != null)
{
foreach (var configToWorkOn in configsToWorkOn)
{
successes.Enqueue(Tuple.Create(configToWorkOn, latestEtag));
}
}
else
{
waitForWork[i] = true;
}
return;
}
var itemsToReplicate = documents.Select(x =>
{
JsonDocument.EnsureIdInMetadata(x);
var doc = x.ToJson();
doc[Constants.DocumentIdFieldName] = x.Key;
return(new ReplicatedDoc
{
Document = doc,
Etag = x.Etag,
Key = x.Key,
SerializedSizeOnDisk = x.SerializedSizeOnDisk
});
}).ToList();
try
{
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, configsToWorkOn, replicationConfig =>
{
try
{
var startTime = SystemTime.UtcNow;
var spRepTime = new Stopwatch();
spRepTime.Start();
var lastReplicatedEtag = replicationConfig.LastReplicatedEtag;
var deletedDocs = deletedDocsByConfig[replicationConfig];
var docsToReplicate = itemsToReplicate
.Where(x => lastReplicatedEtag.CompareTo(x.Etag) < 0) // haven't replicate the etag yet
.Where(document =>
{
var info = Database.Documents.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0)
{
if (Log.IsDebugEnabled)
{
Log.Debug(
"Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
document.Key, replicationConfig.Name, info.TouchedEtag);
}
return(false);
}
}
return(true);
});
if (deletedDocs.Count >= MaxNumberOfDeletionsToReplicate + 1)
{
docsToReplicate = docsToReplicate.Where(x => EtagUtil.IsGreaterThan(x.Etag, deletedDocs[deletedDocs.Count - 1].Etag) == false);
}
var docsToReplicateAsList = docsToReplicate.ToList();
var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicateAsList);
if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicateAsList.Count == 0)
{
currentLatestEtag = lastBatchEtag;
}
int countOfReplicatedItems = 0;
if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) &&
ReplicateChangesToDestination(replicationConfig, docsToReplicateAsList, out countOfReplicatedItems))
{
if (deletedDocs.Count > 0)
{
Database.TransactionalStorage.Batch(accessor =>
accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag));
}
successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag));
}
spRepTime.Stop();
var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick);
var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig);
sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems);
sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems);
sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds);
UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicateAsList.Count);
}
catch (Exception e)
{
Log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Exception = e.ToString(),
Title = "Sql Replication failure to replication",
Message = "Sql Replication could not replicate to " + replicationConfig.Name,
UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name
});
}
});
}
finally
{
prefetchingBehavior.CleanupDocuments(lastBatchEtag);
prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed);
}
}
});
if (successes.Count == 0)
{
if (waitForWork.All(x => x))
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
}
continue;
}
foreach (var t in successes)
{
var cfg = t.Item1;
var currentLatestEtag = t.Item2;
//If a reset was requested we don't want to update the last replicated etag.
//If we do register the success the reset will become a noop.
bool isReset;
if (ResetRequested.TryGetValue(t.Item1.Name, out isReset) && isReset)
{
continue;
}
var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
if (destEtag == null)
{
localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
{
Name = cfg.Name,
LastDocEtag = currentLatestEtag ?? Etag.Empty
});
}
else
{
var lastDocEtag = destEtag.LastDocEtag;
if (currentLatestEtag != null && EtagUtil.IsGreaterThan(currentLatestEtag, lastDocEtag))
{
lastDocEtag = currentLatestEtag;
}
destEtag.LastDocEtag = lastDocEtag;
}
}
//We are done recording success for this batch so we can clear the reset dictionary
ResetRequested.Clear();
SaveNewReplicationStatus(localReplicationStatus);
}
finally
{
AfterReplicationCompleted(successes.Count);
RemoveUnusedPrefetchers(usedPrefetchers);
}
}
}
private async Task <IEnumerable <Task <SynchronizationReport> > > SynchronizeDestinationAsync(SynchronizationDestination destination, bool forceSyncingAll)
{
ICredentials credentials = destination.Credentials;
var destinationSyncClient = new SynchronizationServerClient(destination.ServerUrl, destination.FileSystem, destination.ApiKey, credentials);
bool repeat;
do
{
var lastETag = await destinationSyncClient.GetLastSynchronizationFromAsync(storage.Id).ConfigureAwait(false);
var activeTasks = synchronizationQueue.Active;
var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList();
var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationSyncClient).ConfigureAwait(false);
var needSyncingAgain = new List <FileHeader>();
Debug.Assert(filesNeedConfirmation.Count == confirmations.Length);
for (int i = 0; i < confirmations.Length; i++)
{
var confirmation = confirmations[i];
if (confirmation.Status == FileStatus.Safe)
{
if (Log.IsDebugEnabled)
{
Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName);
}
RemoveSyncingConfiguration(confirmation.FileName, destination.Url);
}
else
{
storage.Batch(accessor =>
{
var fileHeader = accessor.ReadFile(confirmation.FileName);
if (fileHeader == null)
{
if (Log.IsDebugEnabled)
{
Log.Debug("Destination server {0} said that file '{1}' is {2} but such file no longer exists. Removing related syncing configuration", destination, confirmation.FileName, confirmation.Status);
}
RemoveSyncingConfiguration(confirmation.FileName, destination.Url);
}
else if (EtagUtil.IsGreaterThan(fileHeader.Etag, filesNeedConfirmation[i].FileETag))
{
if (Log.IsDebugEnabled)
{
Log.Debug("Destination server {0} said that file '{1}' is {2} but such file has been changed since we stored the syncing configuration. " +
"Stored etag in configuration is {3} while current file etag is {4}. Removing related syncing configuration", destination, confirmation.FileName, confirmation.Status, fileHeader.Etag, filesNeedConfirmation[i].FileETag);
}
RemoveSyncingConfiguration(confirmation.FileName, destination.Url);
}
else
{
needSyncingAgain.Add(fileHeader);
if (Log.IsDebugEnabled)
{
Log.Debug("Destination server {0} said that file '{1}' is {2}.", destination, confirmation.FileName, confirmation.Status);
}
}
});
}
}
if (synchronizationQueue.NumberOfPendingSynchronizationsFor(destination.Url) < AvailableSynchronizationRequestsTo(destination.Url))
{
repeat = await EnqueueMissingUpdatesAsync(destinationSyncClient, lastETag, needSyncingAgain).ConfigureAwait(false) == false;
}
else
{
repeat = false;
}
}while (repeat);
return(SynchronizePendingFilesAsync(destinationSyncClient, forceSyncingAll));
}