public void StartReplication() { var replicationTask = new Task(ReplicationAction, TaskCreationOptions.LongRunning); httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = storage.ReplicationTimeoutInMs }; replicationTask.Start(); }
public IndexReplicationTask(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication) : base(database, httpRavenRequestFactory, replication) { replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min lastQueriedFrequency = TimeSpan.FromSeconds(database.Configuration.TimeToWaitBeforeRunningIdleIndexes.TotalSeconds / 2); TimeToWaitBeforeSendingDeletesOfIndexesToSiblings = TimeSpan.FromMinutes(1); }
public ReplicationTopologyDiscoverer(DocumentDatabase database, RavenJArray @from, int ttl, ILog log) { this.database = database; this.ttl = ttl; this.log = log; this.@from = @from; requestFactory = new HttpRavenRequestFactory(); }
public void Execute(DocumentDatabase database) { docDb = database; var replicationRequestTimeoutInMs = docDb.Configuration.GetConfigurationValue<int>("Raven/Replication/ReplicationRequestTimeout") ?? 7500; httpRavenRequestFactory = new HttpRavenRequestFactory(replicationRequestTimeoutInMs); new Thread(Execute) { IsBackground = true, Name = "Replication Thread" }.Start(); }
public void Execute(DocumentDatabase database) { docDb = database; var replicationRequestTimeoutInMs = docDb.Configuration.GetConfigurationValue<int>("Raven/Replication/ReplicationRequestTimeout") ?? 60 * 1000; httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs }; var task = new Task(Execute, TaskCreationOptions.LongRunning); var disposableAction = new DisposableAction(task.Wait); // make sure that the doc db waits for the replication task shutdown docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction); task.Start(); }
public void Execute(DocumentDatabase database) { docDb = database; var replicationRequestTimeoutInMs = docDb.Configuration.GetConfigurationValue<int>("Raven/Replication/ReplicationRequestTimeout") ?? 7500; httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs }; var thread = new Thread(Execute) { IsBackground = true, Name = "Replication Thread" }; // make sure that the doc db waits for the replication thread shutdown docDb.ExtensionsState.TryAdd(thread, new DisposableAction(thread.Join)); thread.Start(); }
public void Execute(DocumentDatabase database) { etagSynchronizer = database.EtagSynchronizer.GetSynchronizer(EtagSynchronizerType.Replicator); prefetchingBehavior = database.Prefetcher.GetPrefetchingBehavior(PrefetchingUser.Replicator, null); docDb = database; var replicationRequestTimeoutInMs = docDb.Configuration.GetConfigurationValue<int>("Raven/Replication/ReplicationRequestTimeout") ?? 60 * 1000; httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs }; var task = new Task(Execute, TaskCreationOptions.LongRunning); var disposableAction = new DisposableAction(task.Wait); // make sure that the doc db waits for the replication task shutdown docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction); task.Start(); }
public void CompactUpdateMessagesShouldNotAppearInMessages(string requestedStorage) { using (var documentStore = NewRemoteDocumentStore(requestedStorage: requestedStorage, databaseName:"Test",runInMemory:false)) { using (var session = documentStore.OpenSession()) { session.Store(new RavenJObject()); session.SaveChanges(); } var ravenHtttpFactory = new HttpRavenRequestFactory(); var request = ravenHtttpFactory.Create("http://localhost:8079/admin/compact?database=Test","POST",new RavenConnectionStringOptions()); var response = request.ExecuteRequest<RavenJObject>(); using (var systemDocumentStore = new DocumentStore() { Url = "http://localhost:8079" }.Initialize()) { using (var session = systemDocumentStore.OpenSession()) { var stopeWatch = new Stopwatch(); stopeWatch.Start(); do { if (stopeWatch.Elapsed >= timeout) throw new TimeoutException("Waited to long for test to complete compaction."); var statusRequest = ravenHtttpFactory.Create("http://localhost:8079/operation/status?id=" + response.Value<string>("OperationId"), "GET", new RavenConnectionStringOptions()); var status = statusRequest.ExecuteRequest<RavenJObject>(); if (status.Value<bool>("Completed")) { var compactStatus = session.Load<CompactStatus>(CompactStatus.RavenDatabaseCompactStatusDocumentKey("Test")); Assert.Equal(compactStatus.Messages.Count, storageToExpectedLength[requestedStorage]); return; } else if (status.Value<bool>("Faulted")) { throw new Exception("Something went wrong, compaction was not successful"); } Thread.Sleep(1000); } while (true); } } } }
public HttpResponseMessage TransformersReplicate([FromBody] ReplicationDestination replicationDestination) { var op = GetQueryStringValue("op"); if (string.Equals(op, "replicate-all", StringComparison.InvariantCultureIgnoreCase)) return ReplicateAllTransformers(); if (string.Equals(op, "replicate-all-to-destination", StringComparison.InvariantCultureIgnoreCase)) return ReplicateAllTransformers(dest => dest.IsEqualTo(replicationDestination)); var transformerName = GetQueryStringValue("transformerName"); if (transformerName == null) throw new InvalidOperationException("transformerName query string must be specified if op=replicate-all or op=replicate-all-to-destination isn't specified"); HttpResponseMessage erroResponseMessage; var replicationDocument = GetReplicationDocument(out erroResponseMessage); if (replicationDocument == null) return erroResponseMessage; if (string.IsNullOrWhiteSpace(transformerName) || transformerName.StartsWith("/")) return GetMessageWithString(String.Format("Invalid transformer name! Received : '{0}'", transformerName), HttpStatusCode.NotFound); var transformerDefinition = Database.Transformers.GetTransformerDefinition(transformerName); if (transformerDefinition == null) return GetEmptyMessage(HttpStatusCode.NotFound); var clonedTransformerDefinition = transformerDefinition.Clone(); clonedTransformerDefinition.TransfomerId = 0; var serializedTransformerDefinition = RavenJObject.FromObject(clonedTransformerDefinition); var httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds }; var failedDestinations = new ConcurrentBag<string>(); Parallel.ForEach(replicationDocument.Destinations.Where(x => x.Disabled == false && x.SkipIndexReplication == false), destination => ReplicateTransformer(transformerName, destination, serializedTransformerDefinition, failedDestinations, httpRavenRequestFactory)); return GetMessageWithObject(new { SuccessfulReplicationCount = (replicationDocument.Destinations.Count - failedDestinations.Count), FailedDestinationUrls = failedDestinations }); }
public AdminReplicationController() { requestFactory = new HttpRavenRequestFactory(); }
private void ReplicateIndex(string indexName, ReplicationDestination destination, RavenJObject indexDefinition, HttpRavenRequestFactory httpRavenRequestFactory) { var connectionOptions = new RavenConnectionStringOptions { ApiKey = destination.ApiKey, Url = destination.Url, DefaultDatabase = destination.Database }; if (!String.IsNullOrWhiteSpace(destination.Username) && !String.IsNullOrWhiteSpace(destination.Password)) { connectionOptions.Credentials = new NetworkCredential(destination.Username, destination.Password, destination.Domain ?? string.Empty); } const string urlTemplate = "{0}/databases/{1}/indexes/{2}"; var operationUrl = string.Format(urlTemplate, destination.Url, destination.Database, Uri.EscapeUriString(indexName)); var replicationRequest = httpRavenRequestFactory.Create(operationUrl, "PUT", connectionOptions); replicationRequest.Write(indexDefinition); replicationRequest.ExecuteRequest(); }
private void ReplicateTransformer(string transformerName, ReplicationDestination destination, RavenJObject transformerDefinition, ConcurrentBag<string> failedDestinations, HttpRavenRequestFactory httpRavenRequestFactory) { var connectionOptions = new RavenConnectionStringOptions { ApiKey = destination.ApiKey, Url = destination.Url, DefaultDatabase = destination.Database }; if (!String.IsNullOrWhiteSpace(destination.Username) && !String.IsNullOrWhiteSpace(destination.Password)) { connectionOptions.Credentials = new NetworkCredential(destination.Username, destination.Password, destination.Domain ?? string.Empty); } //databases/{databaseName}/transformers/{*id} const string urlTemplate = "{0}/databases/{1}/transformers/{2}"; if (Uri.IsWellFormedUriString(destination.Url, UriKind.RelativeOrAbsolute) == false) { const string error = "Invalid destination URL"; failedDestinations.Add(destination.Url); Log.Error(error); return; } var operationUrl = string.Format(urlTemplate, destination.Url, destination.Database, Uri.EscapeUriString(transformerName)); var replicationRequest = httpRavenRequestFactory.Create(operationUrl, "PUT", connectionOptions); replicationRequest.Write(transformerDefinition); try { replicationRequest.ExecuteRequest(); } catch (Exception e) { Log.ErrorException("failed to replicate index to: " + destination.Url, e); failedDestinations.Add(destination.Url); } }
public TransformerReplicationTask(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication) : base(database, httpRavenRequestFactory, replication) { replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min TimeToWaitBeforeSendingDeletesOfTransformersToSiblings = TimeSpan.FromMinutes(1); }
public void Execute(DocumentDatabase database) { docDb = database; docDb.Notifications.OnIndexChange += OnIndexChange; docDb.Notifications.OnTransformerChange += OnTransformerChange; var replicationRequestTimeoutInMs = docDb.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds; autoTuner = new IndependentBatchSizeAutoTuner(docDb.WorkContext, PrefetchingUser.Replicator); httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs }; var task = new Task(Execute, TaskCreationOptions.LongRunning); var disposableAction = new DisposableAction(task.Wait); // make sure that the doc db waits for the replication task shutdown docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction); _replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min _lastQueriedFrequency = TimeSpan.FromSeconds(database.Configuration.TimeToWaitBeforeRunningIdleIndexes.TotalSeconds / 2); _indexReplicationTaskTimer = database.TimerManager.NewTimer(x => ReplicateIndexesAndTransformersTask(x), TimeSpan.Zero, _replicationFrequency); _lastQueriedTaskTimer = database.TimerManager.NewTimer(SendLastQueriedTask, TimeSpan.Zero, _lastQueriedFrequency); task.Start(); }
public HttpResponseMessage IndexReplicate([FromBody] ReplicationDestination replicationDestination) { var op = GetQueryStringValue("op"); if (string.Equals(op, "replicate-all", StringComparison.InvariantCultureIgnoreCase)) return ReplicateAllIndexes(); if (string.Equals(op, "replicate-all-to-destination", StringComparison.InvariantCultureIgnoreCase)) return ReplicateAllIndexes(dest => dest.IsEqualTo(replicationDestination)); var indexName = GetQueryStringValue("indexName"); if(indexName == null) throw new InvalidOperationException("indexName query string must be specified if op=replicate-all or op=replicate-all-to-destination isn't specified"); //check for replication document before doing work on getting index definitions. //if there is no replication set up --> no point in doing any other work HttpResponseMessage erroResponseMessage; var replicationDocument = GetReplicationDocument(out erroResponseMessage); if (replicationDocument == null) return erroResponseMessage; if (indexName.EndsWith("/")) //since id is part of the url, perhaps a trailing forward slash appears there indexName = indexName.Substring(0, indexName.Length - 1); indexName = HttpUtility.UrlDecode(indexName); var indexDefinition = Database.IndexDefinitionStorage.GetIndexDefinition(indexName); if (indexDefinition == null) { return GetMessageWithObject(new { Message = string.Format("Index with name: {0} not found. Cannot proceed with replication...", indexName) }, HttpStatusCode.NotFound); } var serializedIndexDefinition = RavenJObject.FromObject(indexDefinition); var httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds }; var failedDestinations = new ConcurrentDictionary<string, Exception>(); Parallel.ForEach(replicationDocument.Destinations.Where(dest => dest.Disabled == false && dest.SkipIndexReplication == false), destination => { try { ReplicateIndex(indexName, destination, serializedIndexDefinition, httpRavenRequestFactory); } catch (Exception e) { failedDestinations.TryAdd(destination.Humane ?? "<null?>", e); log.WarnException("Could not replicate index " + indexName + " to " + destination.Humane, e); } }); return GetMessageWithObject(new { SuccessfulReplicationCount = (replicationDocument.Destinations.Count - failedDestinations.Count), FailedDestinationUrls = failedDestinations.Select(x => new { Server = x.Key, Error = x.Value.ToString() }).ToArray() }); }
public void Execute(DocumentDatabase database) { docDb = database; _transactionalStorageId = docDb.TransactionalStorage.Id.ToString(); var replicationRequestTimeoutInMs = docDb.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds; autoTuner = new IndependentBatchSizeAutoTuner(docDb.WorkContext, PrefetchingUser.Replicator); httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs }; var task = new Task(Execute, TaskCreationOptions.LongRunning); var disposableAction = new DisposableAction(task.Wait); // make sure that the doc db waits for the replication task shutdown docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction); IndexReplication = new IndexReplicationTask(database, httpRavenRequestFactory, this); TransformerReplication = new TransformerReplicationTask(database, httpRavenRequestFactory, this); task.Start(); IndexReplication.Start(); TransformerReplication.Start(); }
public RavenDB_3435() { httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Debugger.IsAttached ? 600000 : 15000 }; }
public AdminReplicationInfo() { requestFactory = new HttpRavenRequestFactory(); }
public void Execute(DocumentDatabase database) { docDb = database; var replicationRequestTimeoutInMs = docDb.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds; autoTuner = new IndependentBatchSizeAutoTuner(docDb.WorkContext, PrefetchingUser.Replicator); httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs }; nonBufferedHttpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs, AllowWriteStreamBuffering = false }; var task = new Task(Execute, TaskCreationOptions.LongRunning); var disposableAction = new DisposableAction(task.Wait); // make sure that the doc db waits for the replication task shutdown docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction); task.Start(); }
private HttpResponseMessage ReplicateAllTransformers(Func<ReplicationDestination, bool> destinationPredicate = null) { HttpResponseMessage erroResponseMessage; var replicationDocument = GetReplicationDocument(out erroResponseMessage); if (replicationDocument == null) return erroResponseMessage; var httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds }; var enabledReplicationDestinations = replicationDocument.Destinations .Where(dest => dest.Disabled == false && dest.SkipIndexReplication == false) .ToList(); if (destinationPredicate != null) enabledReplicationDestinations = enabledReplicationDestinations.Where(destinationPredicate).ToList(); if (enabledReplicationDestinations.Count == 0) return GetMessageWithObject(new { Message = "Replication is configured, but no enabled destinations found." }, HttpStatusCode.NotFound); var allTransformerDefinitions = Database.Transformers.Definitions; if (allTransformerDefinitions.Length == 0) return GetMessageWithObject(new { Message = "No transformers to replicate. Nothing to do.. " }); var replicationRequestTasks = new List<Task>(enabledReplicationDestinations.Count * allTransformerDefinitions.Length); var failedDestinations = new ConcurrentBag<string>(); foreach (var definition in allTransformerDefinitions) { var clonedDefinition = definition.Clone(); clonedDefinition.TransfomerId = 0; replicationRequestTasks.AddRange( enabledReplicationDestinations .Select(destination => Task.Run(() => ReplicateTransformer(definition.Name, destination, RavenJObject.FromObject(clonedDefinition), failedDestinations, httpRavenRequestFactory))).ToList()); } Task.WaitAll(replicationRequestTasks.ToArray()); return GetMessageWithObject(new { TransformerCount = allTransformerDefinitions.Length, EnabledDestinationsCount = enabledReplicationDestinations.Count, SuccessfulReplicationCount = ((enabledReplicationDestinations.Count * allTransformerDefinitions.Length) - failedDestinations.Count), FailedDestinationUrls = failedDestinations }); }
private HttpResponseMessage ReplicateAllIndexes(Func<ReplicationDestination, bool> additionalDestinationPredicate = null) { //check for replication document before doing work on getting index definitions. //if there is no replication set up --> no point in doing any other work HttpResponseMessage erroResponseMessage; var replicationDocument = GetReplicationDocument(out erroResponseMessage); if (replicationDocument == null) return erroResponseMessage; var indexDefinitions = Database.IndexDefinitionStorage .IndexDefinitions .Select(x => x.Value) .ToList(); var httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds }; var enabledReplicationDestinations = replicationDocument.Destinations .Where(dest => dest.Disabled == false && dest.SkipIndexReplication == false) .ToList(); if (additionalDestinationPredicate != null) enabledReplicationDestinations = enabledReplicationDestinations.Where(additionalDestinationPredicate).ToList(); if (enabledReplicationDestinations.Count == 0) return GetMessageWithObject(new { Message = "Replication is configured, but no enabled destinations found." }, HttpStatusCode.NotFound); var replicationRequestTasks = new List<Task>(enabledReplicationDestinations.Count * indexDefinitions.Count); var failedDestinations = new ConcurrentDictionary<string, Exception>(); foreach (var definition in indexDefinitions) { replicationRequestTasks.AddRange( enabledReplicationDestinations .Select(destination => Task.Run(() => { try { ReplicateIndex(definition.Name, destination, RavenJObject.FromObject(definition), httpRavenRequestFactory); } catch (Exception e) { failedDestinations.TryAdd(destination.Humane ?? "<null?>", e); log.WarnException("Could not replicate " + definition.Name + " to " + destination.Humane, e); } }))); } Task.WaitAll(replicationRequestTasks.ToArray()); return GetMessageWithObject(new { IndexesCount = indexDefinitions.Count, EnabledDestinationsCount = enabledReplicationDestinations.Count, SuccessfulReplicationCount = ((enabledReplicationDestinations.Count * indexDefinitions.Count) - failedDestinations.Count), FailedDestinationUrls = failedDestinations.Select(x=>new{ Server = x.Key, Error = x.Value.ToString()}).ToArray() }); }
protected ReplicationTaskBase(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication) { Database = database; HttpRavenRequestFactory = httpRavenRequestFactory; Replication = replication; }