public DocumentCount Calculate(ServerInfo serverInfo)
{
var replicationDocument = GetReplicationDocument();
if (serverInfo.SourceId != DatabaseId)
{
return(GetDocumentsLeftCountFromAnotherSourceServer(replicationDocument, serverInfo));
}
var replicationDestination = replicationDocument
.Destinations
.FirstOrDefault(x => FetchTargetServerUrl(x).Equals(serverInfo.DestinationUrl, StringComparison.CurrentCultureIgnoreCase) &&
x.Database.Equals(serverInfo.DatabaseName, StringComparison.CurrentCultureIgnoreCase));
if (replicationDestination == null)
{
throw new InvalidOperationException($"Couldn't find replication destination for url: {serverInfo.DestinationUrl} and database: {serverInfo.DatabaseName}");
}
var replicationStrategy = ReplicationTask.GetConnectionOptions(replicationDestination, database);
if (replicationStrategy.IsETL == false)
{
return(GetDocumentsLeftCount(replicationStrategy, serverInfo.DestinationUrl, serverInfo.DatabaseName));
}
return(GetDocumentsLeftCountForEtl(replicationStrategy, serverInfo.DestinationUrl, serverInfo.DatabaseName));
}
private string FetchTargetServerUrl(ReplicationDestination replicationDestination)
{
var url = $"{replicationDestination.Url}/debug/config";
try
{
var replicationStrategy = ReplicationTask.GetConnectionOptions(replicationDestination, database);
var request = requestFactory.Create(url, HttpMethods.Get, replicationStrategy.ConnectionStringOptions);
var ravenConfig = request.ExecuteRequest <RavenJObject>();
var serverUrlFromTargetConfig = ravenConfig.Value <string>("ServerUrl");
// replace host name with target hostname
return(new UriBuilder(replicationDestination.Url)
{
Host = new Uri(serverUrlFromTargetConfig).Host
}.Uri.ToString());
}
catch (Exception e)
{
if (replicationDestination.Url.EndsWith("/"))
{
return(replicationDestination.Url);
}
return(replicationDestination.Url + "/");
}
}
public void DatabaseMigrationServiceCreateReplicationTask()
{
#region create-replication-task-1481747646288
var response = client.CreateReplicationTask(new CreateReplicationTaskRequest
{
CdcStartTime = new DateTime(2016, 12, 14, 10, 25, 43),
MigrationType = "full-load",
ReplicationInstanceArn = "arn:aws:dms:us-east-1:123456789012:rep:6UTDJGBOUS3VI3SUWA66XFJCJQ",
ReplicationTaskIdentifier = "task1",
ReplicationTaskSettings = "",
SourceEndpointArn = "arn:aws:dms:us-east-1:123456789012:endpoint:ZW5UAN6P4E77EC7YWHK4RZZ3BE",
TableMappings = "file://mappingfile.json",
Tags = new List <Tag> {
new Tag {
Key = "Acount",
Value = "24352226"
}
},
TargetEndpointArn = "arn:aws:dms:us-east-1:123456789012:endpoint:ASXWXJZLNWNT5HTWCGV2BUJQ7E"
});
ReplicationTask replicationTask = response.ReplicationTask;
#endregion
}
public DocumentsLeftToReplicate(DocumentDatabase database)
{
this.database = database;
DatabaseId = database.TransactionalStorage.Id.ToString();
requestFactory = new HttpRavenRequestFactory();
replicationTask = database.StartupTasks.OfType <ReplicationTask>().FirstOrDefault();
if (replicationTask == null)
{
throw new InvalidOperationException("Couldn't locate ReplicationTask");
}
}
private DocumentCount DocumentsLeftCountFromAnotherSourceServer(ServerInfo serverInfo,
ReplicationDestination.ReplicationDestinationWithConfigurationOrigin replicationDestination)
{
var replicationStrategy = ReplicationTask.GetConnectionOptions(replicationDestination, database);
var url = $"{replicationStrategy.ConnectionStringOptions.Url}/admin/replication/docs-left-to-replicate";
var request = requestFactory.Create(url, HttpMethods.Post, replicationStrategy.ConnectionStringOptions);
request.Write(RavenJObject.FromObject(serverInfo));
var documentsLeftCount = request.ExecuteRequest <DocumentCount>();
return(documentsLeftCount);
}
public void DatabaseMigrationServiceStopReplicationTask()
{
#region stop-replication-task-1481762924947
var response = client.StopReplicationTask(new StopReplicationTaskRequest
{
ReplicationTaskArn = "arn:aws:dms:us-east-1:123456789012:endpoint:ASXWXJZLNWNT5HTWCGV2BUJQ7E"
});
ReplicationTask replicationTask = response.ReplicationTask;
#endregion
}
public void DatabaseMigrationServiceDeleteReplicationTask()
{
#region delete-replication-task-1481752903506
var response = client.DeleteReplicationTask(new DeleteReplicationTaskRequest
{
ReplicationTaskArn = "arn:aws:dms:us-east-1:123456789012:rep:6UTDJGBOUS3VI3SUWA66XFJCJQ"
});
ReplicationTask replicationTask = response.ReplicationTask;
#endregion
}
public ReplicationTopologyDiscoverer(DocumentDatabase database, RavenJArray @from, int ttl, ILog log)
{
this.database = database;
this.ttl = ttl;
this.log = log;
this.@from = @from;
requestFactory = new HttpRavenRequestFactory();
currentServerId = database.TransactionalStorage.Id;
replicationTask = database.StartupTasks.OfType <ReplicationTask>().FirstOrDefault();
if (replicationTask == null)
{
throw new InvalidOperationException("Couldn't locate ReplicationTask");
}
}
public void DatabaseMigrationServiceStartReplicationTask()
{
#region start-replication-task-1481762706778
var response = client.StartReplicationTask(new StartReplicationTaskRequest
{
CdcStartTime = new DateTime(2016, 12, 14, 5, 33, 20),
ReplicationTaskArn = "arn:aws:dms:us-east-1:123456789012:rep:6UTDJGBOUS3VI3SUWA66XFJCJQ",
StartReplicationTaskType = "start-replication"
});
ReplicationTask replicationTask = response.ReplicationTask;
#endregion
}
public ReplicationTopologyDiscoverer(DocumentDatabase database, IEnumerable <string> @from, int ttl, ILog log)
{
this.database = database;
this.ttl = ttl;
this.log = log;
this.@from = new HashSet <string>(@from, StringComparer.OrdinalIgnoreCase);
requestFactory = new HttpRavenRequestFactory();
currentServerId = database.TransactionalStorage.Id;
replicationTask = database.StartupTasks.OfType <ReplicationTask>().FirstOrDefault();
if (replicationTask == null)
{
throw new InvalidOperationException("Couldn't locate ReplicationTask");
}
}
public override TimeTicks GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination)
{
var sourceReplicationInformationWithBatchInformation = task.GetLastReplicatedEtagFrom(destination);
if (sourceReplicationInformationWithBatchInformation == null)
{
return(null);
}
if (sourceReplicationInformationWithBatchInformation.LastModified.HasValue == false)
{
return(null);
}
return(new TimeTicks(SystemTime.UtcNow - sourceReplicationInformationWithBatchInformation.LastModified.Value));
}
private ReplicationTopologyDestinationNode HandleDestination(ReplicationDestination replicationDestination)
{
var destination = ReplicationTask.GetConnectionOptions(replicationDestination, database);
if (replicationDestination.Disabled)
{
return(ReplicationTopologyDestinationNode.Disabled(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
}
if (from.Contains(destination.ConnectionStringOptions.Url))
{
var state = CheckDestinationConnectionState(destination);
switch (state)
{
case ReplicatonNodeState.Online:
return(ReplicationTopologyDestinationNode.Online(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
case ReplicatonNodeState.Offline:
return(ReplicationTopologyDestinationNode.Offline(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
default:
throw new NotSupportedException(state.ToString());
}
}
string error;
ReplicationTopologyRootNode rootNode;
if (TryGetSchema(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions, out rootNode, out error))
{
var node = ReplicationTopologyDestinationNode.Online(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
node.Destinations = rootNode.Destinations;
node.Sources = rootNode.Sources;
node.Errors = rootNode.Errors;
return(node);
}
var offline = ReplicationTopologyDestinationNode.Offline(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
if (string.IsNullOrEmpty(error) == false)
{
offline.Errors.Add(error);
}
return(offline);
}
protected override RavenJObject TryGetExisting(string id, out JsonDocument existingItem, out Etag existingEtag, out bool deleted)
{
var existingDoc = Actions.Documents.DocumentByKey(id);
if (existingDoc != null)
{
ReplicationTask.EnsureReplicationInformationInMetadata(existingDoc.Metadata, Database);
if (existingDoc.Metadata.ContainsKey(Constants.LastModified) == false)
{
existingDoc.Metadata.Add(Constants.LastModified, existingDoc.LastModified);
}
existingItem = existingDoc;
existingEtag = existingDoc.Etag;
deleted = false;
return(existingDoc.Metadata);
}
var listItem = Actions.Lists.Read(Constants.RavenReplicationDocsTombstones, id);
if (listItem != null)
{
ReplicationTask.EnsureReplicationInformationInMetadata(listItem.Data, Database);
existingEtag = listItem.Etag;
deleted = true;
existingItem = new JsonDocument
{
Etag = listItem.Etag,
DataAsJson = new RavenJObject(),
Key = listItem.Key,
Metadata = listItem.Data
};
return(listItem.Data);
}
existingEtag = Etag.Empty;
existingItem = null;
deleted = false;
return(null);
}
public void ExtractDocumentIds(ServerInfo serverInfo, Action <string> successAction, Action skippedAction)
{
var replicationDocument = GetReplicationDocument();
var replicationDestination = replicationDocument
.Destinations
.FirstOrDefault(x => FetchTargetServerUrl(x).Equals(serverInfo.DestinationUrl, StringComparison.CurrentCultureIgnoreCase) &&
x.Database.Equals(serverInfo.DatabaseName, StringComparison.CurrentCultureIgnoreCase));
if (replicationDestination == null)
{
throw new InvalidOperationException($"Couldn't find replication destination for url: {serverInfo.DestinationUrl} and database: {serverInfo.DatabaseName}");
}
var replicationStrategy = ReplicationTask.GetConnectionOptions(replicationDestination, database);
var sourcesDocument = replicationTask.GetLastReplicatedEtagFrom(replicationStrategy);
if (sourcesDocument == null)
{
throw new InvalidOperationException($"Couldn't get last replicated etag for " +
$"destination url: {serverInfo.DestinationUrl} and database: {serverInfo.DatabaseName}");
}
database.TransactionalStorage.Batch(actions =>
{
var earlyExit = new Reference <bool>();
//get document count since last replicated etag
var documentIds = actions.Documents.GetDocumentIdsAfterEtag(
sourcesDocument.LastDocumentEtag, int.MaxValue,
WillDocumentBeReplicated(replicationStrategy, sourcesDocument.ServerInstanceId.ToString(), skippedAction),
earlyExit, database.WorkContext.CancellationToken);
foreach (var documentId in documentIds)
{
successAction(documentId);
}
});
}
private static async Task HandleDatabaseInServerAsync(ServerRecord server, string databaseName, IAsyncDatabaseCommands dbCmds, IAsyncDocumentSession session)
{
var databaseRecord = await session.LoadAsync <DatabaseRecord>(server.Id + "/" + databaseName);
if (databaseRecord == null)
{
return;
}
var replicationDocument = await dbCmds.GetAsync(Constants.RavenReplicationDestinations);
if (replicationDocument == null)
{
return;
}
databaseRecord.IsReplicationEnabled = true;
var document = replicationDocument.DataAsJson.JsonDeserialization <ReplicationDocument>();
databaseRecord.ReplicationDestinations = document.Destinations;
var replicationStatistics = await dbCmds.Info.GetReplicationInfoAsync();
if (replicationStatistics != null)
{
databaseRecord.ReplicationStatistics = replicationStatistics;
}
// Monitor the replicated destinations
foreach (var replicationDestination in databaseRecord.ReplicationDestinations)
{
if (replicationDestination.Disabled)
{
continue;
}
var url = replicationDestination.Url;
var databasesIndex = url.IndexOf("/databases/", StringComparison.OrdinalIgnoreCase);
if (databasesIndex > 0)
{
url = url.Substring(0, databasesIndex);
}
var replicationDestinationServer = await session.LoadAsync <ServerRecord>("serverRecords/" + ReplicationTask.EscapeDestinationName(url));
if (replicationDestinationServer == null)
{
replicationDestinationServer = new ServerRecord
{
Url = url,
};
await session.StoreAsync(replicationDestinationServer);
}
else
{
if (DateTimeOffset.UtcNow - server.LastTriedToConnectAt <= TimeSpan.FromHours(1))
{
continue;
}
}
await FetchServerDatabasesAsync(replicationDestinationServer, session);
}
}
private ReplicationTopologyDestinationNode HandleDestination(ReplicationDestination replicationDestination)
{
var destination = ReplicationTask.GetConnectionOptions(replicationDestination, database);
string error;
string targetServerUrl;
// since each server can be addresses using both dns and ips we normalize connection string url by fetching target server url
// it should give us consistent urls
if (FetchTargetServerUrl(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions, out targetServerUrl, out error) == false)
{
var offlineNode = ReplicationTopologyDestinationNode.Offline(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
if (string.IsNullOrEmpty(error) == false)
{
offlineNode.Errors.Add(error);
}
return(offlineNode);
}
if (replicationDestination.Disabled)
{
return(ReplicationTopologyDestinationNode.Disabled(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
}
if (from.Contains(targetServerUrl))
{
var state = CheckDestinationConnectionState(destination);
switch (state)
{
case ReplicatonNodeState.Online:
return(ReplicationTopologyDestinationNode.Online(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
case ReplicatonNodeState.Offline:
return(ReplicationTopologyDestinationNode.Offline(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
default:
throw new NotSupportedException(state.ToString());
}
}
ReplicationTopologyRootNode rootNode;
if (TryGetSchema(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions, out rootNode, out error))
{
var node = ReplicationTopologyDestinationNode.Online(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
node.Destinations = rootNode.Destinations;
node.Sources = rootNode.Sources;
node.Errors = rootNode.Errors;
return(node);
}
var offline = ReplicationTopologyDestinationNode.Offline(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
if (string.IsNullOrEmpty(error) == false)
{
offline.Errors.Add(error);
}
return(offline);
}
public IndexReplicationTask(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication)
: base(database, httpRavenRequestFactory, replication)
{
replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min
lastQueriedFrequency = TimeSpan.FromSeconds(database.Configuration.TimeToWaitBeforeRunningIdleIndexes.TotalSeconds / 2);
TimeToWaitBeforeSendingDeletesOfIndexesToSiblings = TimeSpan.FromMinutes(1);
}
public override void Respond(IHttpContext context)
{
var src = context.Request.QueryString["from"];
if (string.IsNullOrEmpty(src))
{
context.SetStatusToBadRequest();
return;
}
while (src.EndsWith("/"))
{
src = src.Substring(0, src.Length - 1); // remove last /, because that has special meaning for Raven
}
if (string.IsNullOrEmpty(src))
{
context.SetStatusToBadRequest();
return;
}
var array = context.ReadJsonArray();
if (ReplicationTask != null)
{
ReplicationTask.HandleHeartbeat(src);
}
using (Database.DisableAllTriggersForCurrentThread())
{
Database.TransactionalStorage.Batch(actions =>
{
string lastEtag = Guid.Empty.ToString();
foreach (RavenJObject document in array)
{
var metadata = document.Value <RavenJObject>("@metadata");
if (metadata[Constants.RavenReplicationSource] == null)
{
// not sure why, old document from when the user didn't have replication
// that we suddenly decided to replicate, choose the source for that
metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(src);
}
lastEtag = metadata.Value <string>("@etag");
var id = metadata.Value <string>("@id");
document.Remove("@metadata");
ReplicateDocument(actions, id, metadata, document, src);
}
var replicationDocKey = Constants.RavenReplicationSourcesBasePath + "/" + src;
var replicationDocument = Database.Get(replicationDocKey, null);
var lastAttachmentId = Guid.Empty;
if (replicationDocument != null)
{
lastAttachmentId =
replicationDocument.DataAsJson.JsonDeserialization <SourceReplicationInformation>().
LastAttachmentEtag;
}
Guid serverInstanceId;
if (Guid.TryParse(context.Request.QueryString["dbid"], out serverInstanceId) == false)
{
serverInstanceId = Database.TransactionalStorage.Id;
}
Database.Put(replicationDocKey, null,
RavenJObject.FromObject(new SourceReplicationInformation
{
Source = src,
LastDocumentEtag = new Guid(lastEtag),
LastAttachmentEtag = lastAttachmentId,
ServerInstanceId = serverInstanceId
}),
new RavenJObject(), null);
});
}
}
protected override void ConfigureApplicationContainer(TinyIoCContainer container)
{
// Disable the AutoRegister since we're not using it
// base.ConfigureApplicationContainer(container);
var store = new DocumentStore
{
ConnectionStringName = "RavenDB",
};
store.Initialize();
store.Conventions.RegisterIdConvention <ServerRecord>((s, commands, serverRecord) => "serverRecords/" + ReplicationTask.EscapeDestinationName(serverRecord.Url));
store.Conventions.RegisterIdConvention <DatabaseRecord>((s, commands, databaseRecord) => databaseRecord.ServerId + "/" + databaseRecord.Name);
store.Conventions.RegisterAsyncIdConvention <ServerRecord>((s, commands, serverRecord) => new CompletedTask <string>("serverRecords/" + ReplicationTask.EscapeDestinationName(serverRecord.Url)));
store.Conventions.RegisterAsyncIdConvention <DatabaseRecord>((s, commands, databaseRecord) => new CompletedTask <string>(databaseRecord.ServerId + "/" + databaseRecord.Name));
container.Register <IDocumentStore>(store);
}
public async Task <HttpResponseMessage> DocReplicatePost()
{
const int BatchSize = 512;
var src = GetQueryStringValue("from");
if (string.IsNullOrEmpty(src))
{
return(GetEmptyMessage(HttpStatusCode.BadRequest));
}
while (src.EndsWith("/"))
{
src = src.Substring(0, src.Length - 1); // remove last /, because that has special meaning for Raven
}
if (string.IsNullOrEmpty(src))
{
return(GetEmptyMessage(HttpStatusCode.BadRequest));
}
var array = await ReadJsonArrayAsync();
if (ReplicationTask != null)
{
ReplicationTask.HandleHeartbeat(src);
}
using (Database.DisableAllTriggersForCurrentThread())
{
string lastEtag = Etag.Empty.ToString();
var docIndex = 0;
while (docIndex < array.Length)
{
using (Database.DocumentLock.Lock())
{
Database.TransactionalStorage.Batch(actions =>
{
for (var j = 0; j < BatchSize && docIndex < array.Length; j++, docIndex++)
{
var document = (RavenJObject)array[docIndex];
var metadata = document.Value <RavenJObject>("@metadata");
if (metadata[Constants.RavenReplicationSource] == null)
{
// not sure why, old document from when the user didn't have replication
// that we suddenly decided to replicate, choose the source for that
metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(src);
}
lastEtag = metadata.Value <string>("@etag");
var id = metadata.Value <string>("@id");
document.Remove("@metadata");
ReplicateDocument(actions, id, metadata, document, src);
}
SaveReplicationSource(src, lastEtag);
});
}
}
}
return(GetEmptyMessage());
}
public HttpResponseMessage ExplainGet(string docId)
{
if (string.IsNullOrEmpty(docId))
{
return(GetMessageWithString("Document key is required.", HttpStatusCode.BadRequest));
}
var destinationUrl = GetQueryStringValue("destinationUrl");
if (string.IsNullOrEmpty(destinationUrl))
{
return(GetMessageWithString("Destination url is required.", HttpStatusCode.BadRequest));
}
var databaseName = GetQueryStringValue("databaseName");
if (string.IsNullOrEmpty(databaseName))
{
return(GetMessageWithString("Destination database name is required.", HttpStatusCode.BadRequest));
}
var result = new ReplicationExplanationForDocument
{
Key = docId,
Destination = new ReplicationExplanationForDocument.DestinationInformation
{
Url = destinationUrl,
DatabaseName = databaseName
}
};
var destinations = ReplicationTask.GetReplicationDestinations(x => string.Equals(x.Url, destinationUrl, StringComparison.OrdinalIgnoreCase) && string.Equals(x.Database, databaseName, StringComparison.OrdinalIgnoreCase));
if (destinations == null || destinations.Length == 0)
{
result.Message = string.Format("Could not find replication destination for a given url ('{0}') and database ('{1}').", destinationUrl, databaseName);
return(GetMessageWithObject(result));
}
if (destinations.Length > 1)
{
result.Message = string.Format("There is more than one replication destination for a given url ('{0}') and database ('{1}').", destinationUrl, databaseName);
return(GetMessageWithObject(result));
}
var destination = destinations[0];
var destinationsReplicationInformationForSource = ReplicationTask.GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
{
result.Message = "Could not connect to destination server.";
return(GetMessageWithObject(result));
}
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
result.Destination.ServerInstanceId = destinationId;
result.Destination.LastDocumentEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
var document = Database.Documents.Get(docId, null);
if (document == null)
{
result.Message = string.Format("Document with given key ('{0}') does not exist.", docId);
return(GetMessageWithObject(result));
}
result.Key = document.Key;
result.Etag = document.Etag;
string reason;
if (destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) == false)
{
result.Message = reason;
return(GetMessageWithObject(result));
}
reason = EtagUtil.IsGreaterThan(document.Etag, destinationsReplicationInformationForSource.LastDocumentEtag) ? "Document will be replicated." : "Document should have been replicated.";
result.Message = reason;
return(GetMessageWithObject(result));
}
public TransformerReplicationTask(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication)
: base(database, httpRavenRequestFactory, replication)
{
replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min
TimeToWaitBeforeSendingDeletesOfTransformersToSiblings = TimeSpan.FromMinutes(1);
}
public override OctetString GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination)
{
throw new NotSupportedException();
}
public async Task <HttpResponseMessage> DocReplicatePost()
{
const int BatchSize = 512;
var src = GetQueryStringValue("from");
var collections = GetQueryStringValue("collections");
if (string.IsNullOrEmpty(src))
{
return(GetEmptyMessage(HttpStatusCode.BadRequest));
}
while (src.EndsWith("/"))
{
src = src.Substring(0, src.Length - 1);// remove last /, because that has special meaning for Raven
}
if (string.IsNullOrEmpty(src))
{
return(GetEmptyMessage(HttpStatusCode.BadRequest));
}
var array = await ReadJsonArrayAsync().ConfigureAwait(false);
if (ReplicationTask != null)
{
ReplicationTask.HandleHeartbeat(src);
}
using (Database.DisableAllTriggersForCurrentThread())
{
var conflictResolvers = DocsReplicationConflictResolvers;
string lastEtag = Etag.Empty.ToString();
var docIndex = 0;
var retries = 0;
while (retries < 3 && docIndex < array.Length)
{
var lastIndex = docIndex;
using (Database.DocumentLock.Lock())
{
Database.TransactionalStorage.Batch(actions =>
{
for (var j = 0; j < BatchSize && docIndex < array.Length; j++, docIndex++)
{
var document = (RavenJObject)array[docIndex];
var metadata = document.Value <RavenJObject>("@metadata");
if (metadata[Constants.RavenReplicationSource] == null)
{
// not sure why, old document from when the user didn't have replication
// that we suddenly decided to replicate, choose the source for that
metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(src);
}
lastEtag = metadata.Value <string>("@etag");
var id = metadata.Value <string>("@id");
document.Remove("@metadata");
ReplicateDocument(actions, id, metadata, document, src, conflictResolvers);
}
SaveReplicationSource(src, lastEtag, array.Length, collections);
retries = lastIndex == docIndex ? retries : 0;
});
}
if (lastIndex == docIndex)
{
if (retries == 3)
{
Log.Warn("Replication processing did not end up replicating any documents for 3 times in a row, stopping operation", retries);
}
else
{
Log.Warn("Replication processing did not end up replicating any documents, due to possible storage error, retry number: {0}", retries);
}
retries++;
}
}
}
return(GetEmptyMessage());
}
protected ReplicationTaskBase(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication)
{
Database = database;
HttpRavenRequestFactory = httpRavenRequestFactory;
Replication = replication;
}
public override OctetString GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination)
{
return(new OctetString(destination.ConnectionStringOptions.Url.ForDatabase(destination.ConnectionStringOptions.DefaultDatabase)));
}
public abstract TData GetData(DocumentDatabase database, ReplicationTask task, ReplicationStrategy destination);
