public HttpResponseMessage TransformersReplicate([FromBody] ReplicationDestination replicationDestination)
{
var op = GetQueryStringValue("op");
var replicationTask = Database.StartupTasks.OfType <ReplicationTask>().FirstOrDefault();
if (replicationTask == null)
{
return(GetMessageWithString("Could not find replication task. Something is wrong here, check logs for more details", HttpStatusCode.BadRequest));
}
if (string.Equals(op, "replicate-all-to-destination", StringComparison.InvariantCultureIgnoreCase))
{
replicationTask.TransformerReplication.Execute(dest => dest.IsEqualTo(replicationDestination) && dest.SkipIndexReplication == false);
return(GetEmptyMessage());
}
var transformerName = GetQueryStringValue("transformerName");
if (string.IsNullOrEmpty(transformerName) == false)
{
replicationTask.TransformerReplication.Execute(transformerName);
return(GetEmptyMessage());
}
replicationTask.TransformerReplication.Execute();
return(GetEmptyMessage());
}
public ReplicationRule(DeleteMarkerReplication deleteMarkerReplication, ReplicationDestination destination,
ExistingObjectReplication existingObjectReplication, RuleFilter filter, DeleteReplication deleteReplication,
uint priority, string id, string prefix, SourceSelectionCriteria sourceSelectionCriteria, string status)
{
if (string.IsNullOrEmpty(status) || string.IsNullOrWhiteSpace(status))
{
throw new ArgumentNullException(nameof(Status) + " cannot be null or empty.");
}
if (string.IsNullOrEmpty(id) || string.IsNullOrWhiteSpace(id))
{
throw new ArgumentNullException(nameof(ID) + " cannot be null or empty.");
}
if (deleteReplication == null)
{
throw new ArgumentNullException(nameof(DeleteReplication) + " cannot be null or empty.");
}
if (destination == null)
{
throw new ArgumentNullException(nameof(Destination) + " cannot be null or empty.");
}
DeleteMarkerReplication = deleteMarkerReplication;
Destination = destination;
ExistingObjectReplication = existingObjectReplication;
Filter = filter;
Priority = priority;
DeleteReplication = deleteReplication;
ID = id;
Prefix = prefix;
SourceSelectionCriteria = sourceSelectionCriteria;
Status = status;
}
private static RavenConnectionStringOptions GetConnectionOptions(ReplicationDestination x)
{
if (string.IsNullOrEmpty(x.ConnectionStringName))
{
return new RavenConnectionStringOptions
{
Url = x.Url
}
}
;
var connectionStringParser = ConnectionStringParser <RavenConnectionStringOptions> .FromConnectionStringName(x.ConnectionStringName);
connectionStringParser.Parse();
var options = connectionStringParser.ConnectionStringOptions;
if (string.IsNullOrEmpty(options.Url))
{
throw new InvalidOperationException("Could not figure out what the replication URL is");
}
if (string.IsNullOrEmpty(options.DefaultDatabase) == false)
{
if (options.Url.EndsWith("/") == false)
{
options.Url += "/";
}
options.Url += "databases/" + options.DefaultDatabase;
}
return(options);
}
}
private string FetchTargetServerUrl(ReplicationDestination replicationDestination)
{
var url = $"{replicationDestination.Url}/debug/config";
try
{
var replicationStrategy = ReplicationTask.GetConnectionOptions(replicationDestination, database);
var request = requestFactory.Create(url, HttpMethods.Get, replicationStrategy.ConnectionStringOptions);
var ravenConfig = request.ExecuteRequest <RavenJObject>();
var serverUrlFromTargetConfig = ravenConfig.Value <string>("ServerUrl");
// replace host name with target hostname
return(new UriBuilder(replicationDestination.Url)
{
Host = new Uri(serverUrlFromTargetConfig).Host
}.Uri.ToString());
}
catch (Exception e)
{
if (replicationDestination.Url.EndsWith("/"))
{
return(replicationDestination.Url);
}
return(replicationDestination.Url + "/");
}
}
protected void RunReplication(IDocumentStore source, IDocumentStore destination,
TransitiveReplicationOptions transitiveReplicationBehavior = TransitiveReplicationOptions.None,
bool disabled = false,
bool ignoredClient = false,
string apiKey = null)
{
Console.WriteLine("Replicating from {0} to {1}.", source.Url, destination.Url);
using (var session = source.OpenSession())
{
var replicationDestination = new ReplicationDestination
{
Url = destination.Url.Replace("localhost", "ipv4.fiddler"),
TransitiveReplicationBehavior = transitiveReplicationBehavior,
Disabled = disabled,
IgnoredClient = ignoredClient
};
if (apiKey != null)
{
replicationDestination.ApiKey = apiKey;
}
SetupDestination(replicationDestination);
session.Store(new ReplicationDocument
{
Destinations = { replicationDestination }
}, "Raven/Replication/Destinations");
session.SaveChanges();
}
}
protected void RunReplication(IDocumentStore source, IDocumentStore destination,
TransitiveReplicationOptions transitiveReplicationBehavior = TransitiveReplicationOptions.None,
bool disabled = false,
bool ignoredClient = false,
string apiKey = null,
string db = null)
{
Console.WriteLine("Replicating from {0} to {1} with db = {2}.", source.Url, destination.Url, db ?? Constants.SystemDatabase);
using (var session = source.OpenSession(db))
{
var replicationDestination = new ReplicationDestination
{
Url = destination is EmbeddableDocumentStore ?
"http://localhost:" + (destination as EmbeddableDocumentStore).Configuration.Port :
destination.Url.Replace("localhost", "ipv4.fiddler"),
TransitiveReplicationBehavior = transitiveReplicationBehavior,
Disabled = disabled,
IgnoredClient = ignoredClient
};
if (db != null)
{
replicationDestination.Database = db;
}
if (apiKey != null)
{
replicationDestination.ApiKey = apiKey;
}
SetupDestination(replicationDestination);
session.Store(new ReplicationDocument
{
Destinations = { replicationDestination }
}, "Raven/Replication/Destinations");
session.SaveChanges();
}
}
private string GetReplicationUrl(ReplicationDestination replicationDestination)
{
var replicationUrl = replicationDestination.ClientVisibleUrl ?? replicationDestination.Url;
return(string.IsNullOrWhiteSpace(replicationDestination.Database)
? replicationUrl
: replicationUrl + "/databases/" + replicationDestination.Database);
}
private OperationMetadata GetReplicationOperation(ReplicationDestination replicationDestination)
{
var replicationUrl = replicationDestination.ClientVisibleUrl ?? replicationDestination.Url;
var url = string.IsNullOrWhiteSpace(replicationDestination.Database)
? replicationUrl
: replicationUrl + "/databases/" + replicationDestination.Database;
return(new OperationMetadata(url, replicationDestination.Username, replicationDestination.Password, replicationDestination.Domain, replicationDestination.ApiKey));
}
protected override void SetupDestination(ReplicationDestination replicationDestination)
{
replicationDestination.SpecifiedCollections = new Dictionary <string, string>
{
{
"users", @"this.Name = 'patched ' + this.Name;"
}
};
}
private ReplicationStrategy GetConnectionOptions(ReplicationDestination x)
{
var replicationStrategy = new ReplicationStrategy
{
ReplicationOptionsBehavior = x.TransitiveReplicationBehavior,
CurrentDatabaseId = docDb.TransactionalStorage.Id.ToString()
};
return(CreateReplicationStrategyFromDocument(x, replicationStrategy));
}
public long?GetLastReplicatedEtagForDestination(ReplicationDestination dest)
{
foreach (var replicationHandler in _outgoing)
{
if (replicationHandler.Destination.IsMatch(dest))
{
return(replicationHandler._lastSentDocumentEtag);
}
}
return(null);
}
public ReplicationDestination[] GetForDatabase(string databaseName)
{
if (forDatabases.Keys.Contains(databaseName) == false || forDatabases[databaseName] == null)
{
return(null);
}
var result = new ReplicationDestination[forDatabases[databaseName].Count];
forDatabases[databaseName].CopyTo(result);
return(result);
}
public OutgoingReplicationHandler(
DocumentDatabase database,
ReplicationDestination destination)
{
_database = database;
_destination = destination;
_log = LoggingSource.Instance.GetLogger <OutgoingReplicationHandler>(_database.Name);
_database.Notifications.OnDocumentChange += OnDocumentChange;
_database.Notifications.OnIndexChange += OnIndexChange;
_database.Notifications.OnTransformerChange += OnTransformerChange;
_cts = CancellationTokenSource.CreateLinkedTokenSource(_database.DatabaseShutdown);
}
private void AddAndStartOutgoingReplication(ReplicationDestination destination)
{
var outgoingReplication = new OutgoingReplicationHandler(_database, destination);
outgoingReplication.Failed += OnOutgoingSendingFailed;
outgoingReplication.SuccessfulTwoWaysCommunication += OnOutgoingSendingSucceeded;
_outgoing.TryAdd(outgoingReplication); // can't fail, this is a brand new instance
_outgoingFailureInfo.TryAdd(destination, new ConnectionFailureInfo
{
Destination = destination
});
outgoingReplication.Start();
}
public void destination_with_specified_collections_is_not_considered_as_failover_target()
{
var destination = new ReplicationDestination
{
SpecifiedCollections = new Dictionary <string, string>()
{
{
"Orders", null
},
}
};
Assert.False(destination.IgnoredClient);
}
private RavenConnectionStringOptions GetConnectionOptionsSafe(ReplicationDestination x)
{
try
{
return(GetConnectionOptions(x));
}
catch (Exception e)
{
log.ErrorException(
string.Format("IGNORING BAD REPLICATION CONFIG!{0}Could not figure out connection options for [Url: {1}, ConnectionStringName: {2}]",
Environment.NewLine, x.Url, x.ConnectionStringName),
e);
return(null);
}
}
private ReplicationTopologyDestinationNode HandleDestination(ReplicationDestination replicationDestination)
{
var destination = ReplicationTask.GetConnectionOptions(replicationDestination, database);
if (replicationDestination.Disabled)
{
return(ReplicationTopologyDestinationNode.Disabled(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
}
if (from.Contains(destination.ConnectionStringOptions.Url))
{
var state = CheckDestinationConnectionState(destination);
switch (state)
{
case ReplicatonNodeState.Online:
return(ReplicationTopologyDestinationNode.Online(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
case ReplicatonNodeState.Offline:
return(ReplicationTopologyDestinationNode.Offline(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
default:
throw new NotSupportedException(state.ToString());
}
}
string error;
ReplicationTopologyRootNode rootNode;
if (TryGetSchema(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions, out rootNode, out error))
{
var node = ReplicationTopologyDestinationNode.Online(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
node.Destinations = rootNode.Destinations;
node.Sources = rootNode.Sources;
node.Errors = rootNode.Errors;
return(node);
}
var offline = ReplicationTopologyDestinationNode.Offline(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
if (string.IsNullOrEmpty(error) == false)
{
offline.Errors.Add(error);
}
return(offline);
}
private static ReplicationStrategy CreateReplicationStrategyFromDocument(ReplicationDestination x, ReplicationStrategy replicationStrategy)
{
var url = x.Url;
if (string.IsNullOrEmpty(x.Database) == false)
{
url = url + "/databases/" + x.Database;
}
replicationStrategy.ConnectionStringOptions = new RavenConnectionStringOptions
{
Url = url,
ApiKey = x.ApiKey,
Credentials = string.IsNullOrEmpty(x.Domain) ?
new NetworkCredential(x.Username, x.Password) :
new NetworkCredential(x.Username, x.Password, x.Domain),
};
return(replicationStrategy);
}
protected void RunReplication(IDocumentStore source, IDocumentStore destination, TransitiveReplicationOptions transitiveReplicationBehavior = TransitiveReplicationOptions.None)
{
Console.WriteLine("Replicating from {0} to {1}.", source.Url, destination.Url);
using (var session = source.OpenSession())
{
var replicationDestination = new ReplicationDestination
{
Url = destination.Url.Replace("localhost", "ipv4.fiddler"),
TransitiveReplicationBehavior = transitiveReplicationBehavior,
};
SetupDestination(replicationDestination);
session.Store(new ReplicationDocument
{
Destinations = { replicationDestination }
}, "Raven/Replication/Destinations");
session.SaveChanges();
}
}
private void ReplicateTransformer(string transformerName, ReplicationDestination destination, RavenJObject transformerDefinition, ConcurrentBag <string> failedDestinations, HttpRavenRequestFactory httpRavenRequestFactory)
{
var connectionOptions = new RavenConnectionStringOptions
{
ApiKey = destination.ApiKey,
Url = destination.Url,
DefaultDatabase = destination.Database
};
if (!String.IsNullOrWhiteSpace(destination.Username) &&
!String.IsNullOrWhiteSpace(destination.Password))
{
connectionOptions.Credentials = new NetworkCredential(destination.Username, destination.Password, destination.Domain ?? string.Empty);
}
//databases/{databaseName}/transformers/{*id}
const string urlTemplate = "{0}/databases/{1}/transformers/{2}";
if (Uri.IsWellFormedUriString(destination.Url, UriKind.RelativeOrAbsolute) == false)
{
const string error = "Invalid destination URL";
failedDestinations.Add(destination.Url);
Log.Error(error);
return;
}
var operationUrl = string.Format(urlTemplate, destination.Url, destination.Database, Uri.EscapeUriString(transformerName));
var replicationRequest = httpRavenRequestFactory.Create(operationUrl, "PUT", connectionOptions);
replicationRequest.Write(transformerDefinition);
try
{
replicationRequest.ExecuteRequest();
}
catch (Exception e)
{
Log.ErrorException("failed to replicate index to: " + destination.Url, e);
failedDestinations.Add(destination.Url);
}
}
private ReplicationStrategy GetConnectionOptions(ReplicationDestination x)
{
var replicationStrategy = new ReplicationStrategy
{
ReplicationOptionsBehavior = x.TransitiveReplicationBehavior,
CurrentDatabaseId = docDb.TransactionalStorage.Id.ToString()
};
if (string.IsNullOrEmpty(x.ConnectionStringName))
{
replicationStrategy.ConnectionStringOptions = new RavenConnectionStringOptions
{
Url = x.Url
};
return(replicationStrategy);
}
var connectionStringParser = ConnectionStringParser <RavenConnectionStringOptions> .FromConnectionStringName(x.ConnectionStringName);
connectionStringParser.Parse();
var options = connectionStringParser.ConnectionStringOptions;
if (string.IsNullOrEmpty(options.Url))
{
throw new InvalidOperationException("Could not figure out what the replication URL is");
}
if (string.IsNullOrEmpty(options.DefaultDatabase) == false)
{
if (options.Url.EndsWith("/") == false)
{
options.Url += "/";
}
options.Url += "databases/" + options.DefaultDatabase;
}
replicationStrategy.ConnectionStringOptions = options;
return(replicationStrategy);
}
private void ReplicateIndex(string indexName, ReplicationDestination destination, RavenJObject indexDefinition, HttpRavenRequestFactory httpRavenRequestFactory)
{
var connectionOptions = new RavenConnectionStringOptions
{
ApiKey = destination.ApiKey,
Url = destination.Url,
DefaultDatabase = destination.Database
};
if (!String.IsNullOrWhiteSpace(destination.Username) &&
!String.IsNullOrWhiteSpace(destination.Password))
{
connectionOptions.Credentials = new NetworkCredential(destination.Username, destination.Password, destination.Domain ?? string.Empty);
}
const string urlTemplate = "{0}/databases/{1}/indexes/{2}";
var operationUrl = string.Format(urlTemplate, destination.Url, destination.Database, Uri.EscapeUriString(indexName));
var replicationRequest = httpRavenRequestFactory.Create(operationUrl, "PUT", connectionOptions);
replicationRequest.Write(indexDefinition);
replicationRequest.ExecuteRequest();
}
private static ReplicationStrategy CreateReplicationStrategyFromDocument(ReplicationDestination x, ReplicationStrategy replicationStrategy)
{
var url = x.Url;
if (string.IsNullOrEmpty(x.Database) == false)
{
url = url + "/databases/" + x.Database;
}
replicationStrategy.ConnectionStringOptions = new RavenConnectionStringOptions
{
Url = url,
ApiKey = x.ApiKey,
AuthenticationScheme = x.AuthenticationScheme
};
if (string.IsNullOrEmpty(x.Username) == false)
{
replicationStrategy.ConnectionStringOptions.Credentials = string.IsNullOrEmpty(x.Domain)
? new NetworkCredential(x.Username, x.Password)
: new NetworkCredential(x.Username, x.Password, x.Domain);
}
return replicationStrategy;
}
public static ReplicationStrategy GetConnectionOptions(ReplicationDestination x, DocumentDatabase database)
{
var replicationStrategy = new ReplicationStrategy
{
ReplicationOptionsBehavior = x.TransitiveReplicationBehavior,
CurrentDatabaseId = database.TransactionalStorage.Id.ToString()
};
return CreateReplicationStrategyFromDocument(x, replicationStrategy);
}
private static OperationMetadata ConvertReplicationDestinationToOperationMetadata(ReplicationDestination destination, ClusterInformation clusterInformation)
{
var url = string.IsNullOrEmpty(destination.ClientVisibleUrl) ? destination.Url : destination.ClientVisibleUrl;
if (string.IsNullOrEmpty(url) || destination.CanBeFailover() == false)
{
return(null);
}
if (string.IsNullOrEmpty(destination.Database))
{
return(new OperationMetadata(url, destination.Username, destination.Password, destination.Domain, destination.ApiKey, clusterInformation));
}
return(new OperationMetadata(MultiDatabase.GetRootDatabaseUrl(url).ForDatabase(destination.Database), destination.Username, destination.Password, destination.Domain, destination.ApiKey, clusterInformation));
}
private void ReplicateTransformer(string transformerName, ReplicationDestination destination, RavenJObject transformerDefinition, ConcurrentBag<string> failedDestinations, HttpRavenRequestFactory httpRavenRequestFactory)
{
var connectionOptions = new RavenConnectionStringOptions
{
ApiKey = destination.ApiKey,
Url = destination.Url,
DefaultDatabase = destination.Database
};
if (!String.IsNullOrWhiteSpace(destination.Username) &&
!String.IsNullOrWhiteSpace(destination.Password))
{
connectionOptions.Credentials = new NetworkCredential(destination.Username, destination.Password, destination.Domain ?? string.Empty);
}
//databases/{databaseName}/transformers/{*id}
const string urlTemplate = "{0}/databases/{1}/transformers/{2}";
if (Uri.IsWellFormedUriString(destination.Url, UriKind.RelativeOrAbsolute) == false)
{
const string error = "Invalid destination URL";
failedDestinations.Add(destination.Url);
Log.Error(error);
return;
}
var operationUrl = string.Format(urlTemplate, destination.Url, destination.Database, Uri.EscapeUriString(transformerName));
var replicationRequest = httpRavenRequestFactory.Create(operationUrl, "PUT", connectionOptions);
replicationRequest.Write(transformerDefinition);
try
{
replicationRequest.ExecuteRequest();
}
catch (Exception e)
{
Log.ErrorException("failed to replicate index to: " + destination.Url, e);
failedDestinations.Add(destination.Url);
}
}
private void ReplicateIndex(string indexName, ReplicationDestination destination, RavenJObject indexDefinition, HttpRavenRequestFactory httpRavenRequestFactory)
{
var connectionOptions = new RavenConnectionStringOptions
{
ApiKey = destination.ApiKey,
Url = destination.Url,
DefaultDatabase = destination.Database
};
if (!String.IsNullOrWhiteSpace(destination.Username) &&
!String.IsNullOrWhiteSpace(destination.Password))
{
connectionOptions.Credentials = new NetworkCredential(destination.Username, destination.Password, destination.Domain ?? string.Empty);
}
const string urlTemplate = "{0}/databases/{1}/indexes/{2}";
var operationUrl = string.Format(urlTemplate, destination.Url, destination.Database, Uri.EscapeUriString(indexName));
var replicationRequest = httpRavenRequestFactory.Create(operationUrl, "PUT", connectionOptions);
replicationRequest.Write(indexDefinition);
replicationRequest.ExecuteRequest();
}
private void HandleClusterReplicationChanges(List <string> removedNodes, bool enableReplication)
{
var currentTopology = clusterManager.Engine.CurrentTopology;
var replicationDocumentJson = systemDatabase.Documents.Get(Constants.Global.ReplicationDestinationsDocumentName);
var replicationDocument = replicationDocumentJson != null
? replicationDocumentJson.DataAsJson.JsonDeserialization <ReplicationDocument>()
: new ReplicationDocument();
var replicationDocumentNormalizedDestinations = replicationDocument
.Destinations
.ToDictionary(x => RaftHelper.GetNormalizedNodeUrl(x.Url), x => x);
var currentTopologyNormalizedDestionations = currentTopology
.AllNodes
.ToDictionary(x => x.Uri.AbsoluteUri.ToLowerInvariant(), x => x);
var urls = replicationDocumentNormalizedDestinations
.Keys
.Union(currentTopologyNormalizedDestionations.Keys)
.ToList();
foreach (var url in urls)
{
ReplicationDestination destination;
replicationDocumentNormalizedDestinations.TryGetValue(url, out destination);
NodeConnectionInfo node;
currentTopologyNormalizedDestionations.TryGetValue(url, out node);
if (destination == null && node == null)
{
continue; // not possible, but...
}
if (destination != null && node == null)
{
if (removedNodes.Contains(url, StringComparer.OrdinalIgnoreCase) == false)
{
continue; // external destination
}
replicationDocument.Destinations.Remove(destination);
continue;
}
if (string.Equals(node.Name, clusterManager.Engine.Options.SelfConnection.Name, StringComparison.OrdinalIgnoreCase))
{
continue; // skipping self
}
if (destination == null)
{
destination = new ReplicationDestination();
replicationDocument.Destinations.Add(destination);
}
destination.ApiKey = node.ApiKey;
destination.Database = null;
destination.Disabled = enableReplication == false;
destination.Domain = node.Domain;
destination.Password = node.Password;
destination.TransitiveReplicationBehavior = TransitiveReplicationOptions.Replicate;
destination.SkipIndexReplication = false;
destination.Url = node.Uri.AbsoluteUri;
destination.Username = node.Username;
}
systemDatabase.Documents.Put(Constants.Global.ReplicationDestinationsDocumentName, null, RavenJObject.FromObject(replicationDocument), new RavenJObject(), null);
}
private ReplicationStrategy GetConnectionOptions(ReplicationDestination x)
{
var replicationStrategy = new ReplicationStrategy
{
ReplicationOptionsBehavior = x.TransitiveReplicationBehavior,
CurrentDatabaseId = docDb.TransactionalStorage.Id.ToString()
};
if (string.IsNullOrEmpty(x.ConnectionStringName))
{
replicationStrategy.ConnectionStringOptions = new RavenConnectionStringOptions
{
Url = x.Url
};
return replicationStrategy;
}
var connectionStringParser = ConnectionStringParser<RavenConnectionStringOptions>.FromConnectionStringName(x.ConnectionStringName);
connectionStringParser.Parse();
var options = connectionStringParser.ConnectionStringOptions;
if (string.IsNullOrEmpty(options.Url))
throw new InvalidOperationException("Could not figure out what the replication URL is");
if (string.IsNullOrEmpty(options.DefaultDatabase) == false)
{
if (options.Url.EndsWith("/") == false)
options.Url += "/";
options.Url += "databases/" + options.DefaultDatabase;
}
replicationStrategy.ConnectionStringOptions = options;
return replicationStrategy;
}
private RavenConnectionStringOptions GetConnectionOptionsSafe(ReplicationDestination x)
{
try
{
return GetConnectionOptions(x);
}
catch (Exception e)
{
log.ErrorException(
string.Format("IGNORING BAD REPLICATION CONFIG!{0}Could not figure out connection options for [Url: {1}, ConnectionStringName: {2}]",
Environment.NewLine, x.Url, x.ConnectionStringName),
e);
return null;
}
}
private static ReplicationStrategy CreateReplicationStrategyFromConnectionString(ReplicationDestination x, ReplicationStrategy replicationStrategy)
{
var connectionStringParser = ConnectionStringParser <RavenConnectionStringOptions> .FromConnectionStringName(x.ConnectionStringName);
connectionStringParser.Parse();
var options = connectionStringParser.ConnectionStringOptions;
if (string.IsNullOrEmpty(options.Url))
{
throw new InvalidOperationException("Could not figure out what the replication URL is");
}
if (string.IsNullOrEmpty(options.DefaultDatabase) == false)
{
options.Url += "/databases/" + options.DefaultDatabase;
}
replicationStrategy.ConnectionStringOptions = options;
return(replicationStrategy);
}
protected void RunReplication(IDocumentStore source, IDocumentStore destination,
TransitiveReplicationOptions transitiveReplicationBehavior = TransitiveReplicationOptions.None,
bool disabled = false,
bool ignoredClient = false,
string apiKey = null,
string db = null,
string username = null,
string password = null,
string domain = null,
ReplicationClientConfiguration clientConfiguration = null)
{
db = db ?? (destination is DocumentStore ? ((DocumentStore)destination).DefaultDatabase : null);
Console.WriteLine("Replicating from {0} to {1} with db = {2}.", source.Url, destination.Url, db ?? Constants.SystemDatabase);
using (var session = source.OpenSession(db))
{
var replicationDestination = new ReplicationDestination
{
Url = destination is EmbeddableDocumentStore ?
"http://localhost:" + (destination as EmbeddableDocumentStore).Configuration.Port :
destination.Url.Replace("localhost", "ipv4.fiddler"),
TransitiveReplicationBehavior = transitiveReplicationBehavior,
Disabled = disabled,
IgnoredClient = ignoredClient,
SkipIndexReplication = false //precaution
};
if (db != null)
{
replicationDestination.Database = db;
}
if (apiKey != null)
{
replicationDestination.ApiKey = apiKey;
}
if (username != null)
{
replicationDestination.Username = username;
replicationDestination.Password = password;
replicationDestination.Domain = domain;
}
SetupDestination(replicationDestination);
Console.WriteLine("writing rep dests for " + db + " " + source.Url);
session.Store(new ReplicationDocument
{
Destinations = { replicationDestination },
ClientConfiguration = clientConfiguration
}, "Raven/Replication/Destinations");
session.SaveChanges();
}
while (true)
{
using (var s = source.OpenSession(db))
{
var doc = s.Load <ReplicationDocument>("Raven/Replication/Destinations");
if (string.IsNullOrWhiteSpace(doc.Source))
{
Thread.Sleep(100);
continue;
}
break;
}
}
}
private ReplicationStrategy GetConnectionOptions(ReplicationDestination x)
{
var replicationStrategy = new ReplicationStrategy
{
ReplicationOptionsBehavior = x.TransitiveReplicationBehavior,
CurrentDatabaseId = docDb.TransactionalStorage.Id.ToString()
};
return string.IsNullOrEmpty(x.ConnectionStringName) ?
CreateReplicationStrategyFromDocument(x, replicationStrategy) :
CreateReplicationStrategyFromConnectionString(x, replicationStrategy);
}
private static ReplicationStrategy CreateReplicationStrategyFromConnectionString(ReplicationDestination x, ReplicationStrategy replicationStrategy)
{
var connectionStringParser = ConnectionStringParser<RavenConnectionStringOptions>.FromConnectionStringName(x.ConnectionStringName);
connectionStringParser.Parse();
var options = connectionStringParser.ConnectionStringOptions;
if (string.IsNullOrEmpty(options.Url))
throw new InvalidOperationException("Could not figure out what the replication URL is");
if (string.IsNullOrEmpty(options.DefaultDatabase) == false)
{
options.Url += "/databases/" + options.DefaultDatabase;
}
replicationStrategy.ConnectionStringOptions = options;
return replicationStrategy;
}
protected virtual void SetupDestination(ReplicationDestination replicationDestination)
{
}
public HttpResponseMessage IndexReplicate([FromBody] ReplicationDestination replicationDestination)
{
var op = GetQueryStringValue("op");
if (string.Equals(op, "replicate-all", StringComparison.InvariantCultureIgnoreCase))
{
return(ReplicateAllIndexes());
}
if (string.Equals(op, "replicate-all-to-destination", StringComparison.InvariantCultureIgnoreCase))
{
return(ReplicateAllIndexes(dest => dest.IsEqualTo(replicationDestination)));
}
var indexName = GetQueryStringValue("indexName");
if (indexName == null)
{
throw new InvalidOperationException("indexName query string must be specified if op=replicate-all or op=replicate-all-to-destination isn't specified");
}
//check for replication document before doing work on getting index definitions.
//if there is no replication set up --> no point in doing any other work
HttpResponseMessage erroResponseMessage;
var replicationDocument = GetReplicationDocument(out erroResponseMessage);
if (replicationDocument == null)
{
return(erroResponseMessage);
}
if (indexName.EndsWith("/")) //since id is part of the url, perhaps a trailing forward slash appears there
{
indexName = indexName.Substring(0, indexName.Length - 1);
}
indexName = HttpUtility.UrlDecode(indexName);
var indexDefinition = Database.IndexDefinitionStorage.GetIndexDefinition(indexName);
if (indexDefinition == null)
{
return(GetMessageWithObject(new
{
Message = string.Format("Index with name: {0} not found. Cannot proceed with replication...", indexName)
}, HttpStatusCode.NotFound));
}
var serializedIndexDefinition = RavenJObject.FromObject(indexDefinition);
var httpRavenRequestFactory = new HttpRavenRequestFactory {
RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds
};
var failedDestinations = new ConcurrentDictionary <string, Exception>();
Parallel.ForEach(replicationDocument.Destinations.Where(dest => dest.Disabled == false && dest.SkipIndexReplication == false),
destination =>
{
try
{
ReplicateIndex(indexName, destination, serializedIndexDefinition, httpRavenRequestFactory);
}
catch (Exception e)
{
failedDestinations.TryAdd(destination.Humane ?? "<null?>", e);
log.WarnException("Could not replicate index " + indexName + " to " + destination.Humane, e);
}
});
return(GetMessageWithObject(new
{
SuccessfulReplicationCount = (replicationDocument.Destinations.Count - failedDestinations.Count),
FailedDestinationUrls = failedDestinations.Select(x => new { Server = x.Key, Error = x.Value.ToString() }).ToArray()
}));
}
private ReplicationTopologyDestinationNode HandleDestination(ReplicationDestination replicationDestination)
{
var destination = ReplicationTask.GetConnectionOptions(replicationDestination, database);
string error;
string targetServerUrl;
// since each server can be addresses using both dns and ips we normalize connection string url by fetching target server url
// it should give us consistent urls
if (FetchTargetServerUrl(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions, out targetServerUrl, out error) == false)
{
var offlineNode = ReplicationTopologyDestinationNode.Offline(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
if (string.IsNullOrEmpty(error) == false)
offlineNode.Errors.Add(error);
return offlineNode;
}
if (replicationDestination.Disabled)
return ReplicationTopologyDestinationNode.Disabled(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
if (from.Contains(destination.ConnectionStringOptions.Url))
{
var state = CheckDestinationConnectionState(destination);
switch (state)
{
case ReplicatonNodeState.Online:
return ReplicationTopologyDestinationNode.Online(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
case ReplicatonNodeState.Offline:
return ReplicationTopologyDestinationNode.Offline(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
default:
throw new NotSupportedException(state.ToString());
}
}
ReplicationTopologyRootNode rootNode;
if (TryGetSchema(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions, out rootNode, out error))
{
var node = ReplicationTopologyDestinationNode.Online(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
node.Destinations = rootNode.Destinations;
node.Sources = rootNode.Sources;
node.Errors = rootNode.Errors;
return node;
}
var offline = ReplicationTopologyDestinationNode.Offline(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
if (string.IsNullOrEmpty(error) == false)
offline.Errors.Add(error);
return offline;
}
public HttpResponseMessage TransformersReplicate([FromBody] ReplicationDestination replicationDestination)
{
var op = GetQueryStringValue("op");
if (string.Equals(op, "replicate-all", StringComparison.InvariantCultureIgnoreCase))
{
return(ReplicateAllTransformers());
}
if (string.Equals(op, "replicate-all-to-destination", StringComparison.InvariantCultureIgnoreCase))
{
return(ReplicateAllTransformers(dest => dest.IsEqualTo(replicationDestination)));
}
var transformerName = GetQueryStringValue("transformerName");
if (transformerName == null)
{
throw new InvalidOperationException("transformerName query string must be specified if op=replicate-all or op=replicate-all-to-destination isn't specified");
}
HttpResponseMessage erroResponseMessage;
var replicationDocument = GetReplicationDocument(out erroResponseMessage);
if (replicationDocument == null)
{
return(erroResponseMessage);
}
if (string.IsNullOrWhiteSpace(transformerName) || transformerName.StartsWith("/"))
{
return(GetMessageWithString(String.Format("Invalid transformer name! Received : '{0}'", transformerName), HttpStatusCode.NotFound));
}
var transformerDefinition = Database.Transformers.GetTransformerDefinition(transformerName);
if (transformerDefinition == null)
{
return(GetEmptyMessage(HttpStatusCode.NotFound));
}
var clonedTransformerDefinition = transformerDefinition.Clone();
clonedTransformerDefinition.TransfomerId = 0;
var serializedTransformerDefinition = RavenJObject.FromObject(clonedTransformerDefinition);
var httpRavenRequestFactory = new HttpRavenRequestFactory {
RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds
};
var failedDestinations = new ConcurrentBag <string>();
Parallel.ForEach(replicationDocument.Destinations.Where(x => x.Disabled == false && x.SkipIndexReplication == false),
destination => ReplicateTransformer(transformerName, destination, serializedTransformerDefinition, failedDestinations, httpRavenRequestFactory));
return(GetMessageWithObject(new
{
SuccessfulReplicationCount = (replicationDocument.Destinations.Count - failedDestinations.Count),
FailedDestinationUrls = failedDestinations
}));
}
private static RavenConnectionStringOptions GetConnectionOptions(ReplicationDestination x)
{
if (string.IsNullOrEmpty(x.ConnectionStringName))
return new RavenConnectionStringOptions
{
Url = x.Url
};
var connectionStringParser = ConnectionStringParser<RavenConnectionStringOptions>.FromConnectionStringName(x.ConnectionStringName);
connectionStringParser.Parse();
var options = connectionStringParser.ConnectionStringOptions;
if (string.IsNullOrEmpty(options.Url))
throw new InvalidOperationException("Could not figure out what the replication URL is");
if (string.IsNullOrEmpty(options.DefaultDatabase) == false)
{
if (options.Url.EndsWith("/") == false)
options.Url += "/";
options.Url += "databases/" + options.DefaultDatabase;
}
return options;
}
private ReplicationTopologyDestinationNode HandleDestination(ReplicationDestination replicationDestination)
{
var destination = ReplicationTask.GetConnectionOptions(replicationDestination, database);
string error;
string targetServerUrl;
// since each server can be addresses using both dns and ips we normalize connection string url by fetching target server url
// it should give us consistent urls
if (FetchTargetServerUrl(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions, out targetServerUrl, out error) == false)
{
var offlineNode = ReplicationTopologyDestinationNode.Offline(destination.ConnectionStringOptions.Url, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
if (string.IsNullOrEmpty(error) == false)
{
offlineNode.Errors.Add(error);
}
return(offlineNode);
}
if (replicationDestination.Disabled)
{
return(ReplicationTopologyDestinationNode.Disabled(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
}
if (from.Contains(targetServerUrl))
{
var state = CheckDestinationConnectionState(destination);
switch (state)
{
case ReplicatonNodeState.Online:
return(ReplicationTopologyDestinationNode.Online(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
case ReplicatonNodeState.Offline:
return(ReplicationTopologyDestinationNode.Offline(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior));
default:
throw new NotSupportedException(state.ToString());
}
}
ReplicationTopologyRootNode rootNode;
if (TryGetSchema(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions, out rootNode, out error))
{
var node = ReplicationTopologyDestinationNode.Online(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
node.Destinations = rootNode.Destinations;
node.Sources = rootNode.Sources;
node.Errors = rootNode.Errors;
return(node);
}
var offline = ReplicationTopologyDestinationNode.Offline(targetServerUrl, database.TransactionalStorage.Id, destination.ReplicationOptionsBehavior);
if (string.IsNullOrEmpty(error) == false)
{
offline.Errors.Add(error);
}
return(offline);
}
