public void CanReplicateIndex()
{
var requestFactory = new HttpRavenRequestFactory();
using (var sourceServer = GetNewServer(8077))
using (var source = NewRemoteDocumentStore(ravenDbServer: sourceServer))
using (var destinationServer = GetNewServer(8078))
using (var destination = NewRemoteDocumentStore(ravenDbServer: destinationServer))
{
CreateDatabaseWithReplication(source, "testDB");
CreateDatabaseWithReplication(destination, "testDB");
//turn-off automatic index replication - precaution
source.Conventions.IndexAndTransformerReplicationMode = IndexAndTransformerReplicationMode.None;
SetupReplication(source, "testDB", destination);
//make sure not to replicate the index automatically
var userIndex = new UserIndex();
source.DatabaseCommands.ForDatabase("testDB").PutIndex(userIndex.IndexName, userIndex.CreateIndexDefinition());
var indexStatsBeforeReplication = destination.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes;
Assert.False(indexStatsBeforeReplication.Any(index => index.Name.Equals(userIndex.IndexName, StringComparison.InvariantCultureIgnoreCase)));
var replicationRequestUrl = string.Format("{0}/databases/testDB/replication/replicate-indexes?indexName={1}", source.Url, userIndex.IndexName);
var replicationRequest = requestFactory.Create(replicationRequestUrl, "POST", new RavenConnectionStringOptions
{
Url = source.Url
});
replicationRequest.ExecuteRequest();
var indexStatsAfterReplication = destination.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes;
Assert.True(indexStatsAfterReplication.Any(index => index.Name.Equals(userIndex.IndexName, StringComparison.InvariantCultureIgnoreCase)));
}
}
private void ReplicateSingleSideBySideIndex(ReplicationStrategy destination, IndexDefinition indexDefinition, IndexDefinition sideBySideIndexDefinition)
{
var url = $"{destination.ConnectionStringOptions.Url}/replication/side-by-side?{GetDebugInformation()}";
IndexReplaceDocument indexReplaceDocument;
try
{
indexReplaceDocument = Database.Documents.Get(Constants.IndexReplacePrefix + sideBySideIndexDefinition.Name, null).DataAsJson.JsonDeserialization <IndexReplaceDocument>();
}
catch (Exception e)
{
Log.Warn("Cannot get side-by-side index replacement document. Aborting operation. (this exception should not happen and the cause should be investigated)", e);
return;
}
var sideBySideReplicationInfo = new SideBySideReplicationInfo
{
Index = indexDefinition,
SideBySideIndex = sideBySideIndexDefinition,
OriginDatabaseId = destination.CurrentDatabaseId,
IndexReplaceDocument = indexReplaceDocument
};
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethod.Post, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(sideBySideReplicationInfo));
replicationRequest.ExecuteRequest();
}
private void ReplicateTransformerDeletionIfNeeded(List <JsonDocument> transformerTombstones, ReplicationStrategy destination, Dictionary <string, int> replicatedTransformerTombstones)
{
if (transformerTombstones.Count == 0)
{
return;
}
foreach (var tombstone in transformerTombstones)
{
try
{
int value;
if (Database.Transformers.GetTransformerDefinition(tombstone.Key) != null) //if in the meantime the transformer was recreated under the same name
{
replicatedTransformerTombstones.TryGetValue(tombstone.Key, out value);
replicatedTransformerTombstones[tombstone.Key] = value + 1;
continue;
}
var url = string.Format("{0}/transformers/{1}?{2}", destination.ConnectionStringOptions.Url, Uri.EscapeUriString(tombstone.Key), GetDebugInformation());
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethods.Delete, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(EmptyRequestBody));
replicationRequest.ExecuteRequest();
Log.Info("Replicated transformer deletion (transformer name = {0})", tombstone.Key);
replicatedTransformerTombstones.TryGetValue(tombstone.Key, out value);
replicatedTransformerTombstones[tombstone.Key] = value + 1;
}
catch (Exception e)
{
Replication.HandleRequestBufferingErrors(e, destination);
Log.ErrorException(string.Format("Failed to replicate transformer deletion (transformer name = {0})", tombstone.Key), e);
}
}
}
public IndexReplicationTask(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication)
: base(database, httpRavenRequestFactory, replication)
{
replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min
lastQueriedFrequency = TimeSpan.FromSeconds(database.Configuration.TimeToWaitBeforeRunningIdleIndexes.TotalSeconds / 2);
TimeToWaitBeforeSendingDeletesOfIndexesToSiblings = TimeSpan.FromMinutes(1);
}
public SynchronizationTopologyDiscoverer(RavenFileSystem filesystem, RavenJArray @from, int ttl, ILog log)
{
this.filesystem = filesystem;
this.ttl = ttl;
this.log = log;
this.@from = @from;
requestFactory = new HttpRavenRequestFactory();
}
public CountersReplicationTopologyDiscoverer(CounterStorage counterStorage, RavenJArray @from, int ttl, ILog log)
{
this.counterStorage = counterStorage;
this.ttl = ttl;
this.log = log;
this.@from = @from;
requestFactory = new HttpRavenRequestFactory();
}
public TransformerReplicationTask(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication)
: base(database, httpRavenRequestFactory)
{
this.replication = replication;
replicationFrequency = database.Configuration.Replication.IndexAndTransformerReplicationLatency.AsTimeSpan; //by default 10 min
TimeToWaitBeforeSendingDeletesOfTransformersToSiblings = TimeSpan.FromMinutes(1);
}
public ReplicationTopologyDiscoverer(DocumentDatabase database, RavenJArray @from, int ttl, ILog log)
{
this.database = database;
this.ttl = ttl;
this.log = log;
this.@from = @from;
requestFactory = new HttpRavenRequestFactory();
}
public void should_replicate_only_updated_transformer()
{
var requestFactory = new HttpRavenRequestFactory();
using (var sourceServer = GetNewServer(8077))
using (var source = NewRemoteDocumentStore(ravenDbServer: sourceServer, fiddler: true))
using (var destinationServer = GetNewServer(8078))
using (var destination = NewRemoteDocumentStore(ravenDbServer: destinationServer, fiddler: true))
{
CreateDatabaseWithReplication(source, "testDB");
CreateDatabaseWithReplication(destination, "testDB");
source.Conventions.IndexAndTransformerReplicationMode = IndexAndTransformerReplicationMode.None;
destination.Conventions.IndexAndTransformerReplicationMode = IndexAndTransformerReplicationMode.None;
SetupReplication(source, "testDB", store => false, destination);
SetupReplication(destination, "testDB", store => false, source);
for (var i = 0; i < 30; i++)
{
//just for starting the initial index and transformer replication
source.DatabaseCommands.ForDatabase("testDB").Put("test" + i, Etag.Empty, new RavenJObject(), new RavenJObject());
destination.DatabaseCommands.ForDatabase("testDB").Put("test" + (i + 50), Etag.Empty, new RavenJObject(), new RavenJObject());
}
WaitForDocument(destination.DatabaseCommands.ForDatabase("testDB"), "test29");
WaitForDocument(source.DatabaseCommands.ForDatabase("testDB"), "test79");
var userTransformer = new UserWithoutExtraInfoTransformer();
source.DatabaseCommands.ForDatabase("testDB").PutTransformer(userTransformer.TransformerName, userTransformer.CreateTransformerDefinition());
//replicating transformer from the source
var replicationRequestUrl = string.Format("{0}/databases/testDB/replication/replicate-transformers?op=replicate-all", source.Url);
var replicationRequest = requestFactory.Create(replicationRequestUrl, HttpMethods.Post, new RavenConnectionStringOptions
{
Url = source.Url
});
replicationRequest.ExecuteRequest();
var updatedUserTransformer = new UserWithoutExtraInfoTransformer_Extended();
source.DatabaseCommands.ForDatabase("testDB").PutTransformer(userTransformer.TransformerName, updatedUserTransformer.CreateTransformerDefinition());
var transformer = source.DatabaseCommands.ForDatabase("testDB").GetTransformer(userTransformer.TransformerName);
Assert.True(updatedUserTransformer.CreateTransformerDefinition().TransformResults.Equals(transformer.TransformResults));
//replicating transformer from the destination
replicationRequestUrl = string.Format("{0}/databases/testDB/replication/replicate-transformers?op=replicate-all", destination.Url);
replicationRequest = requestFactory.Create(replicationRequestUrl, HttpMethods.Post, new RavenConnectionStringOptions
{
Url = destination.Url
});
replicationRequest.ExecuteRequest();
//the new transformer shouldn't be overwritten
transformer = source.DatabaseCommands.ForDatabase("testDB").GetTransformer(userTransformer.TransformerName);
Assert.True(updatedUserTransformer.CreateTransformerDefinition().TransformResults.Equals(transformer.TransformResults));
}
}
public void StartReplication()
{
var replicationTask = new Task(ReplicationAction, TaskCreationOptions.LongRunning);
httpRavenRequestFactory = new HttpRavenRequestFactory {
RequestTimeoutInMs = storage.ReplicationTimeoutInMs
};
replicationTask.Start();
}
public void Should_replicate_all_transformers_only_to_specific_destination_if_relevant_endpoint_is_hit()
{
var requestFactory = new HttpRavenRequestFactory();
using (var sourceServer = GetNewServer(8077))
using (var source = NewRemoteDocumentStore(ravenDbServer: sourceServer, fiddler: true))
using (var destinationServer1 = GetNewServer(8078))
using (var destination1 = NewRemoteDocumentStore(ravenDbServer: destinationServer1, fiddler: true))
using (var destinationServer2 = GetNewServer())
using (var destination2 = NewRemoteDocumentStore(ravenDbServer: destinationServer2, fiddler: true))
using (var destinationServer3 = GetNewServer(8081))
using (var destination3 = NewRemoteDocumentStore(ravenDbServer: destinationServer3, fiddler: true))
{
CreateDatabaseWithReplication(source, "testDB");
CreateDatabaseWithReplication(destination1, "testDB");
CreateDatabaseWithReplication(destination2, "testDB");
CreateDatabaseWithReplication(destination3, "testDB");
//make sure replication is off for indexes/transformers
source.Conventions.IndexAndTransformerReplicationMode = IndexAndTransformerReplicationMode.None;
var userTransformer = new UserWithoutExtraInfoTransformer();
var anotherTransformer = new AnotherTransformer();
var yetAnotherTransformer = new YetAnotherTransformer();
source.DatabaseCommands.ForDatabase("testDB").PutTransformer(userTransformer.TransformerName, userTransformer.CreateTransformerDefinition());
source.DatabaseCommands.ForDatabase("testDB").PutTransformer(anotherTransformer.TransformerName, anotherTransformer.CreateTransformerDefinition());
source.DatabaseCommands.ForDatabase("testDB").PutTransformer(yetAnotherTransformer.TransformerName, yetAnotherTransformer.CreateTransformerDefinition());
var expectedTransformerNames = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase)
{
userTransformer.TransformerName,
anotherTransformer.TransformerName,
yetAnotherTransformer.TransformerName
};
// ReSharper disable once AccessToDisposedClosure
var destinations = SetupReplication(source, "testDB", store => false, destination1, destination2, destination3);
var replicationRequestUrl = string.Format("{0}/databases/testDB/replication/replicate-transformers?op=replicate-all-to-destination", source.Url);
var replicationRequest = requestFactory.Create(replicationRequestUrl, "POST", new RavenConnectionStringOptions
{
Url = source.Url
});
replicationRequest.Write(RavenJObject.FromObject(destinations[1]));
replicationRequest.ExecuteRequest();
var transformerNamesAtDestination1 = destination1.DatabaseCommands.ForDatabase("testDB").GetTransformers(0, 1024);
var transformerNamesAtDestination2 = destination2.DatabaseCommands.ForDatabase("testDB").GetTransformers(0, 1024);
var transformerNamesAtDestination3 = destination3.DatabaseCommands.ForDatabase("testDB").GetTransformers(0, 1024);
Assert.Equal(0, transformerNamesAtDestination1.Length);
Assert.True(expectedTransformerNames.SetEquals(transformerNamesAtDestination2.Select(x => x.Name).ToArray()));
Assert.Equal(0, transformerNamesAtDestination3.Length);
}
}
public void Replicate_all_transformers_should_respect_disable_replication_flag()
{
var requestFactory = new HttpRavenRequestFactory();
using (var sourceServer = GetNewServer(8077))
using (var source = NewRemoteDocumentStore(ravenDbServer: sourceServer, fiddler: true))
using (var destinationServer1 = GetNewServer(8078))
using (var destination1 = NewRemoteDocumentStore(ravenDbServer: destinationServer1, fiddler: true))
using (var destinationServer2 = GetNewServer())
using (var destination2 = NewRemoteDocumentStore(ravenDbServer: destinationServer2, fiddler: true))
using (var destinationServer3 = GetNewServer(8081))
using (var destination3 = NewRemoteDocumentStore(ravenDbServer: destinationServer3, fiddler: true))
{
CreateDatabaseWithReplication(source, "testDB");
CreateDatabaseWithReplication(destination1, "testDB");
CreateDatabaseWithReplication(destination2, "testDB");
CreateDatabaseWithReplication(destination3, "testDB");
//make sure replication is off for indexes/transformers
source.Conventions.IndexAndTransformerReplicationMode = IndexAndTransformerReplicationMode.None;
var userTransformer = new UserWithoutExtraInfoTransformer();
var anotherTransformer = new AnotherTransformer();
var yetAnotherTransformer = new YetAnotherTransformer();
var conflictDocumentsTransformer = new RavenConflictDocumentsTransformer(); // #RavenDB-3981
source.DatabaseCommands.ForDatabase("testDB").PutTransformer(userTransformer.TransformerName, userTransformer.CreateTransformerDefinition());
source.DatabaseCommands.ForDatabase("testDB").PutTransformer(anotherTransformer.TransformerName, anotherTransformer.CreateTransformerDefinition());
source.DatabaseCommands.ForDatabase("testDB").PutTransformer(yetAnotherTransformer.TransformerName, yetAnotherTransformer.CreateTransformerDefinition());
var expectedTransformerNames = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase)
{
userTransformer.TransformerName,
anotherTransformer.TransformerName,
yetAnotherTransformer.TransformerName,
};
// ReSharper disable once AccessToDisposedClosure
SetupReplication(source, "testDB", store => store == destination2, destination1, destination2, destination3);
var replicationRequestUrl = string.Format("{0}/databases/testDB/replication/replicate-transformers?op=replicate-all", source.Url);
var replicationRequest = requestFactory.Create(replicationRequestUrl, HttpMethods.Post, new RavenConnectionStringOptions
{
Url = source.Url
});
replicationRequest.ExecuteRequest();
var transformerNamesAtDestination1 = destination1.DatabaseCommands.ForDatabase("testDB").GetTransformers(0, 1024).Where(x => x.Name != conflictDocumentsTransformer.TransformerName);
var transformerNamesAtDestination2 = destination2.DatabaseCommands.ForDatabase("testDB").GetTransformers(0, 1024).Where(x => x.Name != conflictDocumentsTransformer.TransformerName);
var transformerNamesAtDestination3 = destination3.DatabaseCommands.ForDatabase("testDB").GetTransformers(0, 1024).Where(x => x.Name != conflictDocumentsTransformer.TransformerName);
Assert.True(expectedTransformerNames.SetEquals(transformerNamesAtDestination1.Select(x => x.Name).ToArray()));
Assert.Equal(0, transformerNamesAtDestination2.Count());
Assert.True(expectedTransformerNames.SetEquals(transformerNamesAtDestination3.Select(x => x.Name).ToArray()));
}
}
private void ReplicateIndexesMultiPut(ReplicationStrategy destination, List <IndexToAdd> indexesToAdd)
{
var serializedIndexDefinitions = RavenJToken.FromObject(indexesToAdd.ToArray());
var url = $"{destination.ConnectionStringOptions.Url}/indexes?{GetDebugInformation()}";
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethods.Put, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(serializedIndexDefinitions);
replicationRequest.ExecuteRequest();
}
public void Replicate_all_indexes_should_respect_disable_indexing_flag()
{
var requestFactory = new HttpRavenRequestFactory();
using (var sourceServer = GetNewServer(9077))
using (var source = NewRemoteDocumentStore(ravenDbServer: sourceServer, fiddler: true))
using (var destinationServer1 = GetNewServer(9078))
using (var destination1 = NewRemoteDocumentStore(ravenDbServer: destinationServer1, fiddler: true))
using (var destinationServer2 = GetNewServer())
using (var destination2 = NewRemoteDocumentStore(ravenDbServer: destinationServer2, fiddler: true))
using (var destinationServer3 = GetNewServer(9081))
using (var destination3 = NewRemoteDocumentStore(ravenDbServer: destinationServer3, fiddler: true))
{
CreateDatabaseWithReplication(source, "testDB");
CreateDatabaseWithReplication(destination1, "testDB");
CreateDatabaseWithReplication(destination2, "testDB");
CreateDatabaseWithReplication(destination3, "testDB");
//turn-off automatic index replication - precaution
source.Conventions.IndexAndTransformerReplicationMode = IndexAndTransformerReplicationMode.None;
// ReSharper disable once AccessToDisposedClosure
//we setup replication so replication to destination2 will have "disabled" flag
SetupReplication(source, "testDB", store => store == destination2, destination1, destination2, destination3);
//make sure not to replicate the index automatically
var userIndex = new UserIndex();
var anotherUserIndex = new AnotherUserIndex();
var yetAnotherUserIndex = new YetAnotherUserIndex();
source.DatabaseCommands.ForDatabase("testDB").PutIndex(userIndex.IndexName, userIndex.CreateIndexDefinition());
source.DatabaseCommands.ForDatabase("testDB").PutIndex(anotherUserIndex.IndexName, anotherUserIndex.CreateIndexDefinition());
source.DatabaseCommands.ForDatabase("testDB").PutIndex(yetAnotherUserIndex.IndexName, yetAnotherUserIndex.CreateIndexDefinition());
var replicationRequestUrl = string.Format("{0}/databases/testDB/replication/replicate-indexes?op=replicate-all", source.Url);
var replicationRequest = requestFactory.Create(replicationRequestUrl, "POST", new RavenConnectionStringOptions
{
Url = source.Url
});
replicationRequest.ExecuteRequest();
var expectedIndexNames = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase)
{
userIndex.IndexName, anotherUserIndex.IndexName, yetAnotherUserIndex.IndexName
};
var indexStatsAfterReplication1 = destination1.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes.Select(x => x.Name);
Assert.True(expectedIndexNames.SetEquals(indexStatsAfterReplication1));
var indexStatsAfterReplication3 = destination3.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes.Select(x => x.Name);
Assert.True(expectedIndexNames.SetEquals(indexStatsAfterReplication3));
//since destination2 has disabled flag - indexes should not replicate to here
var indexStatsAfterReplication2 = destination2.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes.Select(x => x.Name);
Assert.Empty(indexStatsAfterReplication2);
}
}
public void Should_replicate_all_indexes_only_to_specific_destination_if_relevant_endpoint_hit()
{
var requestFactory = new HttpRavenRequestFactory();
using (var sourceServer = GetNewServer(8077))
using (var source = NewRemoteDocumentStore(ravenDbServer: sourceServer))
using (var destinationServer1 = GetNewServer(8078))
using (var destination1 = NewRemoteDocumentStore(ravenDbServer: destinationServer1))
using (var destinationServer2 = GetNewServer())
using (var destination2 = NewRemoteDocumentStore(ravenDbServer: destinationServer2))
using (var destinationServer3 = GetNewServer(8081))
using (var destination3 = NewRemoteDocumentStore(ravenDbServer: destinationServer3))
{
CreateDatabaseWithReplication(source, "testDB");
CreateDatabaseWithReplication(destination1, "testDB");
CreateDatabaseWithReplication(destination2, "testDB");
CreateDatabaseWithReplication(destination3, "testDB");
//turn-off automatic index replication - precaution
source.Conventions.IndexAndTransformerReplicationMode = IndexAndTransformerReplicationMode.None;
// ReSharper disable once AccessToDisposedClosure
var destinationDocuments = SetupReplication(source, "testDB", store => false, destination1, destination2, destination3);
//make sure not to replicate the index automatically
var userIndex = new UserIndex();
var anotherUserIndex = new AnotherUserIndex();
var yetAnotherUserIndex = new YetAnotherUserIndex();
source.DatabaseCommands.ForDatabase("testDB").PutIndex(userIndex.IndexName, userIndex.CreateIndexDefinition());
source.DatabaseCommands.ForDatabase("testDB").PutIndex(anotherUserIndex.IndexName, anotherUserIndex.CreateIndexDefinition());
source.DatabaseCommands.ForDatabase("testDB").PutIndex(yetAnotherUserIndex.IndexName, yetAnotherUserIndex.CreateIndexDefinition());
var replicationRequestUrl = string.Format("{0}/databases/testDB/replication/replicate-indexes?op=replicate-all-to-destination", source.Url);
var replicationRequest = requestFactory.Create(replicationRequestUrl, "POST", new RavenConnectionStringOptions
{
Url = source.Url
});
replicationRequest.Write(RavenJObject.FromObject(destinationDocuments[1]));
replicationRequest.ExecuteRequest();
var expectedIndexNames = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase)
{
userIndex.IndexName, anotherUserIndex.IndexName, yetAnotherUserIndex.IndexName
};
var indexStatsAfterReplication1 = destination1.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes.Select(x => x.Name);
var indexStatsAfterReplication2 = destination2.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes.Select(x => x.Name);
var indexStatsAfterReplication3 = destination3.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes.Select(x => x.Name);
Assert.Equal(0, indexStatsAfterReplication1.Count());
Assert.True(expectedIndexNames.SetEquals(indexStatsAfterReplication2));
Assert.Equal(0, indexStatsAfterReplication3.Count());
}
}
public DocumentsLeftToReplicate(DocumentDatabase database)
{
this.database = database;
DatabaseId = database.TransactionalStorage.Id.ToString();
requestFactory = new HttpRavenRequestFactory();
replicationTask = database.StartupTasks.OfType <ReplicationTask>().FirstOrDefault();
if (replicationTask == null)
{
throw new InvalidOperationException("Couldn't locate ReplicationTask");
}
}
public void SendLastQueried()
{
if (Database.Disposed)
{
return;
}
try
{
using (CultureHelper.EnsureInvariantCulture())
{
var relevantIndexLastQueries = new Dictionary <string, DateTime>();
var relevantIndexes = Database.Statistics.Indexes.Where(indexStats => indexStats.IsInvalidIndex == false && indexStats.Priority != IndexingPriority.Error && indexStats.Priority != IndexingPriority.Disabled && indexStats.LastQueryTimestamp.HasValue);
foreach (var relevantIndex in relevantIndexes)
{
relevantIndexLastQueries[relevantIndex.Name] = relevantIndex.LastQueryTimestamp.GetValueOrDefault();
}
if (relevantIndexLastQueries.Count == 0)
{
return;
}
var destinations = GetReplicationDestinations(x => x.SkipIndexReplication == false);
foreach (var destination in destinations)
{
try
{
string url = destination.ConnectionStringOptions.Url + "/indexes/last-queried";
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethods.Post, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(relevantIndexLastQueries));
replicationRequest.ExecuteRequest();
}
catch (Exception e)
{
Replication.HandleRequestBufferingErrors(e, destination);
Log.WarnException("Could not update last query time of " + destination.ConnectionStringOptions.Url, e);
}
}
}
}
catch (Exception e)
{
Log.ErrorException("Failed to send last queried timestamp of indexes", e);
}
}
public void PutAndDeleteShouldBePropagated(int numberOfNodes)
{
var clusterStores = CreateRaftCluster(numberOfNodes, activeBundles: "Replication", configureStore: store => store.Conventions.FailoverBehavior = FailoverBehavior.ReadFromLeaderWriteToLeader);
SetupClusterConfiguration(clusterStores);
var store1 = clusterStores[0];
servers.ForEach(server =>
{
var sourceDatabase = AsyncHelpers.RunSync(() => server.Server.GetDatabaseInternal(store1.DefaultDatabase));
var sourceReplicationTask = sourceDatabase.StartupTasks.OfType <ReplicationTask>().First();
sourceReplicationTask.IndexReplication.TimeToWaitBeforeSendingDeletesOfIndexesToSiblings = TimeSpan.FromSeconds(0);
});
store1.DatabaseCommands.PutIndex("Test/Index", new Test_Index().CreateIndexDefinition(), true);
var requestFactory = new HttpRavenRequestFactory();
var replicationRequestUrl = string.Format("{0}/replication/replicate-indexes?op=replicate-all", store1.Url.ForDatabase(store1.DefaultDatabase));
if (numberOfNodes > 1)
{
var replicationRequest = requestFactory.Create(replicationRequestUrl, HttpMethod.Post, new RavenConnectionStringOptions {
Url = store1.Url
});
replicationRequest.ExecuteRequest();
}
using (ForceNonClusterRequests(clusterStores))
{
clusterStores.ForEach(store => WaitFor(store.DatabaseCommands, commands => commands.GetIndex("Test/Index") != null, TimeSpan.FromMinutes(1)));
}
store1.DatabaseCommands.DeleteIndex("Test/Index");
if (numberOfNodes > 1)
{
var replicationRequest = requestFactory.Create(replicationRequestUrl, HttpMethod.Post, new RavenConnectionStringOptions {
Url = store1.Url
});
replicationRequest.ExecuteRequest();
}
using (ForceNonClusterRequests(clusterStores))
{
clusterStores.ForEach(store => WaitFor(store.DatabaseCommands, commands => commands.GetIndex("Test/Index") == null, TimeSpan.FromMinutes(1)));
}
}
public ReplicationTopologyDiscoverer(DocumentDatabase database, IEnumerable <string> @from, int ttl, ILog log)
{
this.database = database;
this.ttl = ttl;
this.log = log;
this.@from = new HashSet <string>(@from, StringComparer.OrdinalIgnoreCase);
requestFactory = new HttpRavenRequestFactory();
currentServerId = database.TransactionalStorage.Id;
replicationTask = database.StartupTasks.OfType <ReplicationTask>().FirstOrDefault();
if (replicationTask == null)
{
throw new InvalidOperationException("Couldn't locate ReplicationTask");
}
}
public ReplicationTopologyDiscoverer(DocumentDatabase database, RavenJArray @from, int ttl, ILog log)
{
this.database = database;
this.ttl = ttl;
this.log = log;
this.@from = @from;
requestFactory = new HttpRavenRequestFactory();
currentServerId = database.TransactionalStorage.Id;
replicationTask = database.StartupTasks.OfType <ReplicationTask>().FirstOrDefault();
if (replicationTask == null)
{
throw new InvalidOperationException("Couldn't locate ReplicationTask");
}
}
private ReplicationInfoStatus[] CheckDestinations(ReplicationDocument replicationDocument)
{
var results = new ReplicationInfoStatus[replicationDocument.Destinations.Count];
Parallel.ForEach(replicationDocument.Destinations, (replicationDestination, state, i) =>
{
var url = replicationDestination.Url;
if (!url.ToLower().Contains("/databases/"))
{
url += "/databases/" + replicationDestination.Database;
}
var result = new ReplicationInfoStatus
{
Url = url,
Status = "Valid",
Code = (int)HttpStatusCode.OK
};
results[i] = result;
var ravenConnectionStringOptions = new RavenConnectionStringOptions
{
ApiKey = replicationDestination.ApiKey,
DefaultDatabase = replicationDestination.Database,
};
if (string.IsNullOrEmpty(replicationDestination.Username) == false)
{
ravenConnectionStringOptions.Credentials = new NetworkCredential(replicationDestination.Username,
replicationDestination.Password,
replicationDestination.Domain ?? string.Empty);
}
var requestFactory = new HttpRavenRequestFactory();
var request = requestFactory.Create(url + "/replication/info", HttpMethods.Post, ravenConnectionStringOptions);
try
{
request.ExecuteRequest();
}
catch (WebException e)
{
FillStatus(result, e);
}
});
return(results);
}
public void CompactUpdateMessagesShouldNotAppearInMessages(string requestedStorage)
{
using (var documentStore = NewRemoteDocumentStore(requestedStorage: requestedStorage, databaseName: "Test", runInMemory: false))
{
using (var session = documentStore.OpenSession())
{
session.Store(new RavenJObject());
session.SaveChanges();
}
var ravenHtttpFactory = new HttpRavenRequestFactory();
var request = ravenHtttpFactory.Create("http://localhost:8079/admin/compact?database=Test", "POST", new RavenConnectionStringOptions());
var response = request.ExecuteRequest <RavenJObject>();
using (var systemDocumentStore = new DocumentStore()
{
Url = "http://localhost:8079"
}.Initialize())
{
using (var session = systemDocumentStore.OpenSession())
{
var stopeWatch = new Stopwatch();
stopeWatch.Start();
do
{
if (stopeWatch.Elapsed >= timeout)
{
throw new TimeoutException("Waited to long for test to complete compaction.");
}
var statusRequest = ravenHtttpFactory.Create("http://localhost:8079/operation/status?id=" + response.Value <string>("OperationId"), "GET", new RavenConnectionStringOptions());
var status = statusRequest.ExecuteRequest <RavenJObject>();
if (status.Value <bool>("Completed"))
{
var compactStatus = session.Load <CompactStatus>(CompactStatus.RavenDatabaseCompactStatusDocumentKey("Test"));
Assert.Equal(compactStatus.Messages.Count, storageToExpectedLength[requestedStorage]);
return;
}
else if (status.Value <bool>("Faulted"))
{
throw new Exception("Something went wrong, compaction was not successful");
}
Thread.Sleep(1000);
} while (true);
}
}
}
}
private void ReplicateIndexDeletionIfNeeded(
List <JsonDocument> indexTombstones,
ReplicationStrategy destination,
Dictionary <string, int> replicatedIndexTombstones)
{
if (indexTombstones.Count == 0)
{
return;
}
foreach (var tombstone in indexTombstones)
{
try
{
int value;
//In case the index was recreated under the same name we will increase the destination count for this tombstone
//As if we sent the delete request but without actually sending the request, ending with a NOOP and deleting the index tombstone.
if (Database.IndexStorage.HasIndex(tombstone.Key))
{
replicatedIndexTombstones.TryGetValue(tombstone.Key, out value);
replicatedIndexTombstones[tombstone.Key] = value + 1;
continue;
}
var url = string.Format("{0}/indexes/{1}?{2}&{3}",
destination.ConnectionStringOptions.Url,
Uri.EscapeUriString(tombstone.Key),
GetTombstoneVersion(tombstone, IndexDefinitionStorage.IndexVersionKey, Constants.IndexVersion),
GetDebugInformation());
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethods.Delete, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(EmptyRequestBody));
replicationRequest.ExecuteRequest();
Log.Info("Replicated index deletion (index name = {0})", tombstone.Key);
replicatedIndexTombstones.TryGetValue(tombstone.Key, out value);
replicatedIndexTombstones[tombstone.Key] = value + 1;
}
catch (Exception e)
{
Replication.HandleRequestBufferingErrors(e, destination);
Log.ErrorException(string.Format("Failed to replicate index deletion (index name = {0})", tombstone.Key), e);
}
}
}
public void Should_skip_index_replication_if_serverside_flag_is_true()
{
var requestFactory = new HttpRavenRequestFactory();
using (var sourceServer = GetNewServer(8077))
using (var source = NewRemoteDocumentStore(ravenDbServer: sourceServer))
using (var destinationServer1 = GetNewServer(8078))
using (var destination1 = NewRemoteDocumentStore(ravenDbServer: destinationServer1))
using (var destinationServer2 = GetNewServer())
using (var destination2 = NewRemoteDocumentStore(ravenDbServer: destinationServer2))
using (var destinationServer3 = GetNewServer(8081))
using (var destination3 = NewRemoteDocumentStore(ravenDbServer: destinationServer3))
{
CreateDatabaseWithReplication(source, "testDB");
CreateDatabaseWithReplication(destination1, "testDB");
CreateDatabaseWithReplication(destination2, "testDB");
CreateDatabaseWithReplication(destination3, "testDB");
//turn-off automatic index replication - precaution
source.Conventions.IndexAndTransformerReplicationMode = IndexAndTransformerReplicationMode.None;
// ReSharper disable once AccessToDisposedClosure
SetupReplication(source, "testDB", store => store == destination2, destination1, destination2, destination3);
//make sure not to replicate the index automatically
var userIndex = new UserIndex();
source.DatabaseCommands.ForDatabase("testDB").PutIndex(userIndex.IndexName, userIndex.CreateIndexDefinition());
var replicationRequestUrl = string.Format("{0}/databases/testDB/replication/replicate-indexes?op=replication&indexName={1}", source.Url, userIndex.IndexName);
var replicationRequest = requestFactory.Create(replicationRequestUrl, "POST", new RavenConnectionStringOptions
{
Url = source.Url
});
replicationRequest.ExecuteRequest();
var indexStatsAfterReplication = destination1.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes;
Assert.True(indexStatsAfterReplication.Any(index => index.Name.Equals(userIndex.IndexName, StringComparison.InvariantCultureIgnoreCase)));
//this one should not have replicated index -> because of SkipIndexReplication = true in ReplicationDocument of source
indexStatsAfterReplication = destination2.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes;
Assert.False(indexStatsAfterReplication.Any(index => index.Name.Equals(userIndex.IndexName, StringComparison.InvariantCultureIgnoreCase)));
indexStatsAfterReplication = destination3.DatabaseCommands.ForDatabase("testDB").GetStatistics().Indexes;
Assert.True(indexStatsAfterReplication.Any(index => index.Name.Equals(userIndex.IndexName, StringComparison.InvariantCultureIgnoreCase)));
}
}
private void ReplicateSingleTransformer(ReplicationStrategy destination, TransformerDefinition definition)
{
try
{
var clonedTransformer = definition.Clone();
clonedTransformer.TransfomerId = 0;
var url = destination.ConnectionStringOptions.Url + "/transformers/" + Uri.EscapeUriString(definition.Name) + "?" + GetDebugInformation();
var replicationRequest = HttpRavenRequestFactory.Create(url, HttpMethod.Put, destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(clonedTransformer));
replicationRequest.ExecuteRequest();
}
catch (Exception e)
{
Replication.HandleRequestBufferingErrors(e, destination);
Log.WarnException("Could not replicate transformer " + definition.Name + " to " + destination.ConnectionStringOptions.Url, e);
}
}
public void Execute(DocumentDatabase database)
{
docDb = database;
var replicationRequestTimeoutInMs =
docDb.Configuration.GetConfigurationValue <int>("Raven/Replication/ReplicationRequestTimeout") ?? 7500;
httpRavenRequestFactory = new HttpRavenRequestFactory {
RequestTimeoutInMs = replicationRequestTimeoutInMs
};
var thread = new Thread(Execute)
{
IsBackground = true,
Name = "Replication Thread"
};
// make sure that the doc db waits for the replication thread shutdown
docDb.ExtensionsState.TryAdd(thread, new DisposableAction(thread.Join));
thread.Start();
}
public void Execute(DocumentDatabase database)
{
etagSynchronizer = database.EtagSynchronizer.GetSynchronizer(EtagSynchronizerType.Replicator);
prefetchingBehavior = database.Prefetcher.GetPrefetchingBehavior(PrefetchingUser.Replicator);
docDb = database;
var replicationRequestTimeoutInMs =
docDb.Configuration.GetConfigurationValue <int>("Raven/Replication/ReplicationRequestTimeout") ??
60 * 1000;
httpRavenRequestFactory = new HttpRavenRequestFactory {
RequestTimeoutInMs = replicationRequestTimeoutInMs
};
var task = new Task(Execute, TaskCreationOptions.LongRunning);
var disposableAction = new DisposableAction(task.Wait);
// make sure that the doc db waits for the replication task shutdown
docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction);
task.Start();
}
protected void SetupClusterConfiguration(List <DocumentStore> clusterStores, bool enableReplication = true, Dictionary <string, string> databaseSettings = null)
{
var clusterStore = clusterStores[0];
var requestFactory = new HttpRavenRequestFactory();
var replicationRequestUrl = string.Format("{0}/admin/cluster/commands/configuration", clusterStore.Url);
var replicationRequest = requestFactory.Create(replicationRequestUrl, HttpMethod.Put, new RavenConnectionStringOptions
{
Url = clusterStore.Url
});
replicationRequest.Write(RavenJObject.FromObject(new ClusterConfiguration
{
EnableReplication = enableReplication,
DatabaseSettings = databaseSettings
}));
replicationRequest.ExecuteRequest();
clusterStores.ForEach(store => WaitForDocument(store.DatabaseCommands.ForSystemDatabase(), Constants.Global.ReplicationDestinationsDocumentName));
using (ForceNonClusterRequests(clusterStores))
{
clusterStores.ForEach(store => WaitFor(store.DatabaseCommands, commands =>
{
using (var request = commands.CreateRequest("/configuration/replication", HttpMethod.Get))
{
var replicationDocumentJson = request.ReadResponseJson() as RavenJObject;
if (replicationDocumentJson == null)
{
return(false);
}
var replicationDocument = replicationDocumentJson.JsonDeserialization <ReplicationDocument>();
return(replicationDocument.Destinations.Count == clusterStores.Count - 1);
}
}));
}
}
private void ReplicateTransformer(string transformerName, ReplicationDestination destination, RavenJObject transformerDefinition, ConcurrentBag <string> failedDestinations, HttpRavenRequestFactory httpRavenRequestFactory)
{
var connectionOptions = new RavenConnectionStringOptions
{
ApiKey = destination.ApiKey,
Url = destination.Url,
DefaultDatabase = destination.Database
};
if (!String.IsNullOrWhiteSpace(destination.Username) &&
!String.IsNullOrWhiteSpace(destination.Password))
{
connectionOptions.Credentials = new NetworkCredential(destination.Username, destination.Password, destination.Domain ?? string.Empty);
}
//databases/{databaseName}/transformers/{*id}
const string urlTemplate = "{0}/databases/{1}/transformers/{2}";
if (Uri.IsWellFormedUriString(destination.Url, UriKind.RelativeOrAbsolute) == false)
{
const string error = "Invalid destination URL";
failedDestinations.Add(destination.Url);
Log.Error(error);
return;
}
var operationUrl = string.Format(urlTemplate, destination.Url, destination.Database, Uri.EscapeUriString(transformerName));
var replicationRequest = httpRavenRequestFactory.Create(operationUrl, "PUT", connectionOptions);
replicationRequest.Write(transformerDefinition);
try
{
replicationRequest.ExecuteRequest();
}
catch (Exception e)
{
Log.ErrorException("failed to replicate index to: " + destination.Url, e);
failedDestinations.Add(destination.Url);
}
}
private void ReplicateIndex(string indexName, ReplicationDestination destination, RavenJObject indexDefinition, HttpRavenRequestFactory httpRavenRequestFactory)
{
var connectionOptions = new RavenConnectionStringOptions
{
ApiKey = destination.ApiKey,
Url = destination.Url,
DefaultDatabase = destination.Database
};
if (!String.IsNullOrWhiteSpace(destination.Username) &&
!String.IsNullOrWhiteSpace(destination.Password))
{
connectionOptions.Credentials = new NetworkCredential(destination.Username, destination.Password, destination.Domain ?? string.Empty);
}
const string urlTemplate = "{0}/databases/{1}/indexes/{2}";
var operationUrl = string.Format(urlTemplate, destination.Url, destination.Database, Uri.EscapeUriString(indexName));
var replicationRequest = httpRavenRequestFactory.Create(operationUrl, "PUT", connectionOptions);
replicationRequest.Write(indexDefinition);
replicationRequest.ExecuteRequest();
}
