public void StartReplication()
{
var replicationTask = new Task(ReplicationAction, TaskCreationOptions.LongRunning);
httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = storage.ReplicationTimeoutInMs };
replicationTask.Start();
}
public IndexReplicationTask(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication)
: base(database, httpRavenRequestFactory, replication)
{
replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min
lastQueriedFrequency = TimeSpan.FromSeconds(database.Configuration.TimeToWaitBeforeRunningIdleIndexes.TotalSeconds / 2);
TimeToWaitBeforeSendingDeletesOfIndexesToSiblings = TimeSpan.FromMinutes(1);
}
public ReplicationTopologyDiscoverer(DocumentDatabase database, RavenJArray @from, int ttl, ILog log)
{
this.database = database;
this.ttl = ttl;
this.log = log;
this.@from = @from;
requestFactory = new HttpRavenRequestFactory();
}
public void Execute(DocumentDatabase database)
{
docDb = database;
var replicationRequestTimeoutInMs =
docDb.Configuration.GetConfigurationValue<int>("Raven/Replication/ReplicationRequestTimeout") ?? 7500;
httpRavenRequestFactory = new HttpRavenRequestFactory(replicationRequestTimeoutInMs);
new Thread(Execute)
{
IsBackground = true,
Name = "Replication Thread"
}.Start();
}
public void Execute(DocumentDatabase database)
{
docDb = database;
var replicationRequestTimeoutInMs =
docDb.Configuration.GetConfigurationValue<int>("Raven/Replication/ReplicationRequestTimeout") ??
60 * 1000;
httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs };
var task = new Task(Execute, TaskCreationOptions.LongRunning);
var disposableAction = new DisposableAction(task.Wait);
// make sure that the doc db waits for the replication task shutdown
docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction);
task.Start();
}
public void Execute(DocumentDatabase database)
{
docDb = database;
var replicationRequestTimeoutInMs =
docDb.Configuration.GetConfigurationValue<int>("Raven/Replication/ReplicationRequestTimeout") ?? 7500;
httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs };
var thread = new Thread(Execute)
{
IsBackground = true,
Name = "Replication Thread"
};
// make sure that the doc db waits for the replication thread shutdown
docDb.ExtensionsState.TryAdd(thread, new DisposableAction(thread.Join));
thread.Start();
}
public void Execute(DocumentDatabase database)
{
etagSynchronizer = database.EtagSynchronizer.GetSynchronizer(EtagSynchronizerType.Replicator);
prefetchingBehavior = database.Prefetcher.GetPrefetchingBehavior(PrefetchingUser.Replicator, null);
docDb = database;
var replicationRequestTimeoutInMs =
docDb.Configuration.GetConfigurationValue<int>("Raven/Replication/ReplicationRequestTimeout") ??
60 * 1000;
httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs };
var task = new Task(Execute, TaskCreationOptions.LongRunning);
var disposableAction = new DisposableAction(task.Wait);
// make sure that the doc db waits for the replication task shutdown
docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction);
task.Start();
}
public void CompactUpdateMessagesShouldNotAppearInMessages(string requestedStorage)
{
using (var documentStore = NewRemoteDocumentStore(requestedStorage: requestedStorage, databaseName:"Test",runInMemory:false))
{
using (var session = documentStore.OpenSession())
{
session.Store(new RavenJObject());
session.SaveChanges();
}
var ravenHtttpFactory = new HttpRavenRequestFactory();
var request = ravenHtttpFactory.Create("http://localhost:8079/admin/compact?database=Test","POST",new RavenConnectionStringOptions());
var response = request.ExecuteRequest<RavenJObject>();
using (var systemDocumentStore = new DocumentStore() { Url = "http://localhost:8079" }.Initialize())
{
using (var session = systemDocumentStore.OpenSession())
{
var stopeWatch = new Stopwatch();
stopeWatch.Start();
do
{
if (stopeWatch.Elapsed >= timeout) throw new TimeoutException("Waited to long for test to complete compaction.");
var statusRequest = ravenHtttpFactory.Create("http://localhost:8079/operation/status?id=" + response.Value<string>("OperationId"), "GET", new RavenConnectionStringOptions());
var status = statusRequest.ExecuteRequest<RavenJObject>();
if (status.Value<bool>("Completed"))
{
var compactStatus = session.Load<CompactStatus>(CompactStatus.RavenDatabaseCompactStatusDocumentKey("Test"));
Assert.Equal(compactStatus.Messages.Count, storageToExpectedLength[requestedStorage]);
return;
} else if (status.Value<bool>("Faulted"))
{
throw new Exception("Something went wrong, compaction was not successful");
}
Thread.Sleep(1000);
} while (true);
}
}
}
}
public HttpResponseMessage TransformersReplicate([FromBody] ReplicationDestination replicationDestination)
{
var op = GetQueryStringValue("op");
if (string.Equals(op, "replicate-all", StringComparison.InvariantCultureIgnoreCase))
return ReplicateAllTransformers();
if (string.Equals(op, "replicate-all-to-destination", StringComparison.InvariantCultureIgnoreCase))
return ReplicateAllTransformers(dest => dest.IsEqualTo(replicationDestination));
var transformerName = GetQueryStringValue("transformerName");
if (transformerName == null)
throw new InvalidOperationException("transformerName query string must be specified if op=replicate-all or op=replicate-all-to-destination isn't specified");
HttpResponseMessage erroResponseMessage;
var replicationDocument = GetReplicationDocument(out erroResponseMessage);
if (replicationDocument == null)
return erroResponseMessage;
if (string.IsNullOrWhiteSpace(transformerName) || transformerName.StartsWith("/"))
return GetMessageWithString(String.Format("Invalid transformer name! Received : '{0}'", transformerName), HttpStatusCode.NotFound);
var transformerDefinition = Database.Transformers.GetTransformerDefinition(transformerName);
if (transformerDefinition == null)
return GetEmptyMessage(HttpStatusCode.NotFound);
var clonedTransformerDefinition = transformerDefinition.Clone();
clonedTransformerDefinition.TransfomerId = 0;
var serializedTransformerDefinition = RavenJObject.FromObject(clonedTransformerDefinition);
var httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds };
var failedDestinations = new ConcurrentBag<string>();
Parallel.ForEach(replicationDocument.Destinations.Where(x => x.Disabled == false && x.SkipIndexReplication == false),
destination => ReplicateTransformer(transformerName, destination, serializedTransformerDefinition, failedDestinations, httpRavenRequestFactory));
return GetMessageWithObject(new
{
SuccessfulReplicationCount = (replicationDocument.Destinations.Count - failedDestinations.Count),
FailedDestinationUrls = failedDestinations
});
}
public AdminReplicationController()
{
requestFactory = new HttpRavenRequestFactory();
}
private void ReplicateIndex(string indexName, ReplicationDestination destination, RavenJObject indexDefinition, HttpRavenRequestFactory httpRavenRequestFactory)
{
var connectionOptions = new RavenConnectionStringOptions
{
ApiKey = destination.ApiKey,
Url = destination.Url,
DefaultDatabase = destination.Database
};
if (!String.IsNullOrWhiteSpace(destination.Username) &&
!String.IsNullOrWhiteSpace(destination.Password))
{
connectionOptions.Credentials = new NetworkCredential(destination.Username, destination.Password, destination.Domain ?? string.Empty);
}
const string urlTemplate = "{0}/databases/{1}/indexes/{2}";
var operationUrl = string.Format(urlTemplate, destination.Url, destination.Database, Uri.EscapeUriString(indexName));
var replicationRequest = httpRavenRequestFactory.Create(operationUrl, "PUT", connectionOptions);
replicationRequest.Write(indexDefinition);
replicationRequest.ExecuteRequest();
}
private void ReplicateTransformer(string transformerName, ReplicationDestination destination, RavenJObject transformerDefinition, ConcurrentBag<string> failedDestinations, HttpRavenRequestFactory httpRavenRequestFactory)
{
var connectionOptions = new RavenConnectionStringOptions
{
ApiKey = destination.ApiKey,
Url = destination.Url,
DefaultDatabase = destination.Database
};
if (!String.IsNullOrWhiteSpace(destination.Username) &&
!String.IsNullOrWhiteSpace(destination.Password))
{
connectionOptions.Credentials = new NetworkCredential(destination.Username, destination.Password, destination.Domain ?? string.Empty);
}
//databases/{databaseName}/transformers/{*id}
const string urlTemplate = "{0}/databases/{1}/transformers/{2}";
if (Uri.IsWellFormedUriString(destination.Url, UriKind.RelativeOrAbsolute) == false)
{
const string error = "Invalid destination URL";
failedDestinations.Add(destination.Url);
Log.Error(error);
return;
}
var operationUrl = string.Format(urlTemplate, destination.Url, destination.Database, Uri.EscapeUriString(transformerName));
var replicationRequest = httpRavenRequestFactory.Create(operationUrl, "PUT", connectionOptions);
replicationRequest.Write(transformerDefinition);
try
{
replicationRequest.ExecuteRequest();
}
catch (Exception e)
{
Log.ErrorException("failed to replicate index to: " + destination.Url, e);
failedDestinations.Add(destination.Url);
}
}
public TransformerReplicationTask(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication)
: base(database, httpRavenRequestFactory, replication)
{
replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min
TimeToWaitBeforeSendingDeletesOfTransformersToSiblings = TimeSpan.FromMinutes(1);
}
public void Execute(DocumentDatabase database)
{
docDb = database;
docDb.Notifications.OnIndexChange += OnIndexChange;
docDb.Notifications.OnTransformerChange += OnTransformerChange;
var replicationRequestTimeoutInMs = docDb.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds;
autoTuner = new IndependentBatchSizeAutoTuner(docDb.WorkContext, PrefetchingUser.Replicator);
httpRavenRequestFactory = new HttpRavenRequestFactory
{
RequestTimeoutInMs = replicationRequestTimeoutInMs
};
var task = new Task(Execute, TaskCreationOptions.LongRunning);
var disposableAction = new DisposableAction(task.Wait);
// make sure that the doc db waits for the replication task shutdown
docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction);
_replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min
_lastQueriedFrequency = TimeSpan.FromSeconds(database.Configuration.TimeToWaitBeforeRunningIdleIndexes.TotalSeconds / 2);
_indexReplicationTaskTimer = database.TimerManager.NewTimer(x => ReplicateIndexesAndTransformersTask(x), TimeSpan.Zero, _replicationFrequency);
_lastQueriedTaskTimer = database.TimerManager.NewTimer(SendLastQueriedTask, TimeSpan.Zero, _lastQueriedFrequency);
task.Start();
}
public HttpResponseMessage IndexReplicate([FromBody] ReplicationDestination replicationDestination)
{
var op = GetQueryStringValue("op");
if (string.Equals(op, "replicate-all", StringComparison.InvariantCultureIgnoreCase))
return ReplicateAllIndexes();
if (string.Equals(op, "replicate-all-to-destination", StringComparison.InvariantCultureIgnoreCase))
return ReplicateAllIndexes(dest => dest.IsEqualTo(replicationDestination));
var indexName = GetQueryStringValue("indexName");
if(indexName == null)
throw new InvalidOperationException("indexName query string must be specified if op=replicate-all or op=replicate-all-to-destination isn't specified");
//check for replication document before doing work on getting index definitions.
//if there is no replication set up --> no point in doing any other work
HttpResponseMessage erroResponseMessage;
var replicationDocument = GetReplicationDocument(out erroResponseMessage);
if (replicationDocument == null)
return erroResponseMessage;
if (indexName.EndsWith("/")) //since id is part of the url, perhaps a trailing forward slash appears there
indexName = indexName.Substring(0, indexName.Length - 1);
indexName = HttpUtility.UrlDecode(indexName);
var indexDefinition = Database.IndexDefinitionStorage.GetIndexDefinition(indexName);
if (indexDefinition == null)
{
return GetMessageWithObject(new
{
Message = string.Format("Index with name: {0} not found. Cannot proceed with replication...", indexName)
}, HttpStatusCode.NotFound);
}
var serializedIndexDefinition = RavenJObject.FromObject(indexDefinition);
var httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds };
var failedDestinations = new ConcurrentDictionary<string, Exception>();
Parallel.ForEach(replicationDocument.Destinations.Where(dest => dest.Disabled == false && dest.SkipIndexReplication == false),
destination =>
{
try
{
ReplicateIndex(indexName, destination, serializedIndexDefinition, httpRavenRequestFactory);
}
catch (Exception e)
{
failedDestinations.TryAdd(destination.Humane ?? "<null?>", e);
log.WarnException("Could not replicate index " + indexName + " to " + destination.Humane, e);
}
});
return GetMessageWithObject(new
{
SuccessfulReplicationCount = (replicationDocument.Destinations.Count - failedDestinations.Count),
FailedDestinationUrls = failedDestinations.Select(x => new { Server = x.Key, Error = x.Value.ToString() }).ToArray()
});
}
public void Execute(DocumentDatabase database)
{
docDb = database;
_transactionalStorageId = docDb.TransactionalStorage.Id.ToString();
var replicationRequestTimeoutInMs = docDb.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds;
autoTuner = new IndependentBatchSizeAutoTuner(docDb.WorkContext, PrefetchingUser.Replicator);
httpRavenRequestFactory = new HttpRavenRequestFactory
{
RequestTimeoutInMs = replicationRequestTimeoutInMs
};
var task = new Task(Execute, TaskCreationOptions.LongRunning);
var disposableAction = new DisposableAction(task.Wait);
// make sure that the doc db waits for the replication task shutdown
docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction);
IndexReplication = new IndexReplicationTask(database, httpRavenRequestFactory, this);
TransformerReplication = new TransformerReplicationTask(database, httpRavenRequestFactory, this);
task.Start();
IndexReplication.Start();
TransformerReplication.Start();
}
public RavenDB_3435()
{
httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Debugger.IsAttached ? 600000 : 15000 };
}
public AdminReplicationInfo()
{
requestFactory = new HttpRavenRequestFactory();
}
public void Execute(DocumentDatabase database)
{
docDb = database;
var replicationRequestTimeoutInMs = docDb.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds;
autoTuner = new IndependentBatchSizeAutoTuner(docDb.WorkContext, PrefetchingUser.Replicator);
httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs };
nonBufferedHttpRavenRequestFactory = new HttpRavenRequestFactory
{
RequestTimeoutInMs = replicationRequestTimeoutInMs,
AllowWriteStreamBuffering = false
};
var task = new Task(Execute, TaskCreationOptions.LongRunning);
var disposableAction = new DisposableAction(task.Wait);
// make sure that the doc db waits for the replication task shutdown
docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction);
task.Start();
}
private HttpResponseMessage ReplicateAllTransformers(Func<ReplicationDestination, bool> destinationPredicate = null)
{
HttpResponseMessage erroResponseMessage;
var replicationDocument = GetReplicationDocument(out erroResponseMessage);
if (replicationDocument == null)
return erroResponseMessage;
var httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds };
var enabledReplicationDestinations = replicationDocument.Destinations
.Where(dest => dest.Disabled == false && dest.SkipIndexReplication == false)
.ToList();
if (destinationPredicate != null)
enabledReplicationDestinations = enabledReplicationDestinations.Where(destinationPredicate).ToList();
if (enabledReplicationDestinations.Count == 0)
return GetMessageWithObject(new { Message = "Replication is configured, but no enabled destinations found." }, HttpStatusCode.NotFound);
var allTransformerDefinitions = Database.Transformers.Definitions;
if (allTransformerDefinitions.Length == 0)
return GetMessageWithObject(new { Message = "No transformers to replicate. Nothing to do.. " });
var replicationRequestTasks = new List<Task>(enabledReplicationDestinations.Count * allTransformerDefinitions.Length);
var failedDestinations = new ConcurrentBag<string>();
foreach (var definition in allTransformerDefinitions)
{
var clonedDefinition = definition.Clone();
clonedDefinition.TransfomerId = 0;
replicationRequestTasks.AddRange(
enabledReplicationDestinations
.Select(destination =>
Task.Run(() =>
ReplicateTransformer(definition.Name, destination,
RavenJObject.FromObject(clonedDefinition),
failedDestinations,
httpRavenRequestFactory))).ToList());
}
Task.WaitAll(replicationRequestTasks.ToArray());
return GetMessageWithObject(new
{
TransformerCount = allTransformerDefinitions.Length,
EnabledDestinationsCount = enabledReplicationDestinations.Count,
SuccessfulReplicationCount = ((enabledReplicationDestinations.Count * allTransformerDefinitions.Length) - failedDestinations.Count),
FailedDestinationUrls = failedDestinations
});
}
private HttpResponseMessage ReplicateAllIndexes(Func<ReplicationDestination, bool> additionalDestinationPredicate = null)
{
//check for replication document before doing work on getting index definitions.
//if there is no replication set up --> no point in doing any other work
HttpResponseMessage erroResponseMessage;
var replicationDocument = GetReplicationDocument(out erroResponseMessage);
if (replicationDocument == null)
return erroResponseMessage;
var indexDefinitions = Database.IndexDefinitionStorage
.IndexDefinitions
.Select(x => x.Value)
.ToList();
var httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds };
var enabledReplicationDestinations = replicationDocument.Destinations
.Where(dest => dest.Disabled == false && dest.SkipIndexReplication == false)
.ToList();
if (additionalDestinationPredicate != null)
enabledReplicationDestinations = enabledReplicationDestinations.Where(additionalDestinationPredicate).ToList();
if (enabledReplicationDestinations.Count == 0)
return GetMessageWithObject(new { Message = "Replication is configured, but no enabled destinations found." }, HttpStatusCode.NotFound);
var replicationRequestTasks = new List<Task>(enabledReplicationDestinations.Count * indexDefinitions.Count);
var failedDestinations = new ConcurrentDictionary<string, Exception>();
foreach (var definition in indexDefinitions)
{
replicationRequestTasks.AddRange(
enabledReplicationDestinations
.Select(destination =>
Task.Run(() =>
{
try
{
ReplicateIndex(definition.Name, destination,
RavenJObject.FromObject(definition),
httpRavenRequestFactory);
}
catch (Exception e)
{
failedDestinations.TryAdd(destination.Humane ?? "<null?>", e);
log.WarnException("Could not replicate " + definition.Name + " to " + destination.Humane, e);
}
})));
}
Task.WaitAll(replicationRequestTasks.ToArray());
return GetMessageWithObject(new
{
IndexesCount = indexDefinitions.Count,
EnabledDestinationsCount = enabledReplicationDestinations.Count,
SuccessfulReplicationCount = ((enabledReplicationDestinations.Count * indexDefinitions.Count) - failedDestinations.Count),
FailedDestinationUrls = failedDestinations.Select(x=>new{ Server = x.Key, Error = x.Value.ToString()}).ToArray()
});
}
protected ReplicationTaskBase(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication)
{
Database = database;
HttpRavenRequestFactory = httpRavenRequestFactory;
Replication = replication;
}
